ctxora 6.2.0

package/src/compact/handoff.py

@@ -0,0 +1,179 @@
+"""Persistent, no-compression conversation handoff records."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sqlite3
+import time
+import uuid
+
+from chunking.treesitter_chunker import count_tokens
+
+DEFAULT_HANDOFF_THRESHOLD_TOKENS = 30_000
+
+
+def _detect_format(messages: list[dict]) -> str:
+    """Detect provider message formats without importing the retired compactor."""
+    if not messages:
+        return "unknown"
+    if any("parts" in message for message in messages):
+        return "gemini"
+    if any(isinstance(message.get("content"), list) for message in messages):
+        return "anthropic"
+    return "openai"
+
+
+class ConversationHandoffStore:
+    """Store original conversation payloads for a client-created fresh task."""
+
+    def __init__(self, db_path: str | None = None):
+        self.db_path = (
+            db_path
+            or os.environ.get("CTXORA_HANDOFF_DB")
+            or os.environ.get("MCP_HARNESS_HANDOFF_DB")
+            or os.path.expanduser("~/.ctxora/handoffs.sqlite3")
+        )
+        if self.db_path != ":memory:":
+            os.makedirs(os.path.dirname(os.path.abspath(self.db_path)), exist_ok=True)
+        self._conn = sqlite3.connect(self.db_path, timeout=5)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA busy_timeout=5000")
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS conversation_handoffs_v2 (
+                handoff_id TEXT PRIMARY KEY,
+                workspace_id TEXT NOT NULL DEFAULT 'legacy_global',
+                source_hash TEXT NOT NULL,
+                provider TEXT NOT NULL,
+                messages_json TEXT NOT NULL,
+                token_count INTEGER NOT NULL,
+                label TEXT NOT NULL DEFAULT '',
+                created_at REAL NOT NULL,
+                expires_at REAL,
+                UNIQUE(workspace_id, source_hash)
+            )
+        """)
+        if self._table_exists("conversation_handoffs"):
+            self._conn.execute("""
+                INSERT OR IGNORE INTO conversation_handoffs_v2 (
+                    handoff_id, workspace_id, source_hash, provider,
+                    messages_json, token_count, label, created_at
+                )
+                SELECT handoff_id, 'legacy_global', source_hash, provider,
+                       messages_json, token_count, label, created_at
+                FROM conversation_handoffs
+            """)
+        self._conn.commit()
+
+    def _table_exists(self, name: str) -> bool:
+        return self._conn.execute(
+            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)
+        ).fetchone() is not None
+
+    @staticmethod
+    def _parse_messages(messages_json: str) -> list[dict]:
+        try:
+            messages = json.loads(messages_json)
+        except (json.JSONDecodeError, ValueError) as error:
+            raise ValueError("messages_json must be a JSON array") from error
+        if not isinstance(messages, list) or not all(isinstance(item, dict) for item in messages):
+            raise ValueError("messages_json must be a JSON array of message objects")
+        return messages
+
+    def prepare(
+        self,
+        messages_json: str,
+        threshold_tokens: int = DEFAULT_HANDOFF_THRESHOLD_TOKENS,
+        label: str = "",
+        workspace_id: str = "legacy_global",
+        retention_seconds: int = 604_800,
+        consent: bool = True,
+    ) -> dict:
+        """Return continue below threshold or persist an exact handoff payload."""
+        messages = self._parse_messages(messages_json)
+        token_count = count_tokens(messages_json)
+        threshold_tokens = max(1, int(threshold_tokens))
+        provider = _detect_format(messages)
+        base = {
+            "history_tokens": token_count,
+            "threshold_tokens": threshold_tokens,
+            "provider": provider,
+            "message_count": len(messages),
+        }
+        if token_count < threshold_tokens:
+            return {"action": "continue", **base}
+        if not consent:
+            raise ValueError("explicit consent is required before storing raw conversations")
+
+        source_hash = hashlib.sha256(messages_json.encode("utf-8")).hexdigest()
+        existing = self._conn.execute(
+            "SELECT handoff_id, created_at FROM conversation_handoffs_v2 WHERE workspace_id = ? AND source_hash = ?",
+            (workspace_id, source_hash),
+        ).fetchone()
+        if existing is None:
+            handoff_id = str(uuid.uuid4())
+            created_at = time.time()
+            self._conn.execute("""
+                INSERT INTO conversation_handoffs_v2 (
+                    handoff_id, workspace_id, source_hash, provider, messages_json,
+                    token_count, label, created_at, expires_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (
+                handoff_id, workspace_id, source_hash, provider, messages_json,
+                token_count, label, created_at, created_at + max(1, retention_seconds),
+            ))
+            self._conn.commit()
+        else:
+            handoff_id = existing["handoff_id"]
+            created_at = existing["created_at"]
+
+        return {
+            "action": "handoff",
+            "handoff_id": handoff_id,
+            "created_at": created_at,
+            "preserved_without_compaction": True,
+            "next_tool": "restore_conversation_handoff",
+            **base,
+        }
+
+    def restore(self, handoff_id: str, workspace_id: str = "legacy_global") -> dict | None:
+        """Return the original, unmodified JSON payload for an explicit restore."""
+        row = self._conn.execute(
+            "SELECT * FROM conversation_handoffs_v2 WHERE handoff_id = ? AND workspace_id = ? AND (expires_at IS NULL OR expires_at > ?)", (handoff_id, workspace_id, time.time())
+        ).fetchone()
+        if row is None:
+            return None
+        return {
+            "handoff_id": row["handoff_id"],
+            "workspace_id": row["workspace_id"],
+            "provider": row["provider"],
+            "history_tokens": row["token_count"],
+            "label": row["label"],
+            "created_at": row["created_at"],
+            "messages_json": row["messages_json"],
+        }
+
+    def list(self, workspace_id: str = "legacy_global", limit: int = 30) -> list[dict]:
+        rows = self._conn.execute(
+            "SELECT handoff_id, provider, token_count, label, created_at, expires_at FROM conversation_handoffs_v2 WHERE workspace_id = ? ORDER BY created_at DESC LIMIT ?",
+            (workspace_id, max(0, limit)),
+        ).fetchall()
+        return [dict(row) for row in rows]
+
+    def delete(self, handoff_id: str, workspace_id: str = "legacy_global") -> bool:
+        cursor = self._conn.execute(
+            "DELETE FROM conversation_handoffs_v2 WHERE handoff_id = ? AND workspace_id = ?",
+            (handoff_id, workspace_id),
+        )
+        self._conn.commit()
+        return cursor.rowcount > 0
+
+    def purge_expired(self) -> int:
+        cursor = self._conn.execute(
+            "DELETE FROM conversation_handoffs_v2 WHERE expires_at IS NOT NULL AND expires_at <= ?",
+            (time.time(),),
+        )
+        self._conn.commit()
+        return cursor.rowcount

package/src/compact/openai.py

@@ -0,0 +1,318 @@
+"""
+compact/openai.py — OpenAI / Codex format compaction
+=====================================================
+Handles both Chat Completions format and legacy Completions format.
+
+Supported model families:
+  - gpt-4o, gpt-4, gpt-3.5-turbo  (chat)
+  - o1, o3, o3-mini, o4-mini       (reasoning)
+  - codex (code-davinci-002, etc.)  (legacy completions → lifted to chat)
+  - gpt-4.1, gpt-4.5               (latest chat)
+
+Message roles recognised:
+  system | user | assistant | tool | function | developer
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+# ── Helpers ──────────────────────────────────────────────────────────────────
+
+def _extract_text(content: Any) -> str:
+    """
+    Flatten OpenAI content to plain text.
+
+    OpenAI content can be:
+      - str
+      - list[dict]  (multi-modal: [{type: "text", text: "..."}, {type: "image_url", ...}])
+    """
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for part in content:
+            if isinstance(part, dict):
+                t = part.get("type", "")
+                if t == "text":
+                    parts.append(part.get("text", ""))
+                elif t == "image_url":
+                    parts.append("[image]")
+                elif t == "input_audio":
+                    parts.append("[audio]")
+                elif t == "refusal":
+                    parts.append(f"[refusal: {part.get('refusal', '')}]")
+        return " ".join(parts)
+    return str(content) if content is not None else ""
+
+
+def _is_openai_format(messages: list[dict]) -> bool:
+    """
+    Heuristic: at least one message has 'role' key matching OpenAI roles.
+    """
+    openai_roles = {"system", "user", "assistant", "tool",
+                    "function", "developer"}
+    for m in messages[:5]:
+        if isinstance(m, dict) and m.get("role") in openai_roles:
+            return True
+    return False
+
+
+def _summarize_chunk(messages: list[dict]) -> str:
+    """
+    Build a structured summary of the omitted message chunk.
+    Extracts user intents + assistant key responses without calling an LLM.
+    """
+    user_intents: list[str] = []
+    assistant_actions: list[str] = []
+    tool_calls_seen: list[str] = []
+    code_snippets: int = 0
+
+    for m in messages:
+        role = m.get("role", "")
+        text = _extract_text(m.get("content") or "").strip()
+
+        if role == "system":
+            continue  # system prompts handled separately
+
+        elif role in ("user", "developer"):
+            if text:
+                # Condense: first sentence or first 120 chars
+                first = re.split(r"[.!?\n]", text)[0].strip()
+                if first:
+                    user_intents.append(first[:120])
+
+        elif role == "assistant":
+            # Count code blocks as signals
+            code_snippets += len(re.findall(r"```", text)) // 2
+
+            # Tool calls in structured format
+            tool_calls = m.get("tool_calls") or []
+            for tc in tool_calls:
+                fn = tc.get("function", {}).get("name", "")
+                if fn:
+                    tool_calls_seen.append(fn)
+
+            if text and not tool_calls:
+                first = re.split(r"[.!?\n]", text)[0].strip()
+                if first:
+                    assistant_actions.append(first[:120])
+
+        elif role == "tool":
+            # Tool responses: just note the tool name
+            name = m.get("name") or ""
+            if name:
+                tool_calls_seen.append(f"{name}→result")
+
+        elif role == "function":
+            name = m.get("name") or ""
+            if name:
+                tool_calls_seen.append(f"{name}→result")
+
+    lines: list[str] = []
+    if user_intents:
+        lines.append(f"User requests: {'; '.join(user_intents[:5])}")
+    if assistant_actions:
+        lines.append(f"Assistant actions: {'; '.join(assistant_actions[:5])}")
+    if tool_calls_seen:
+        unique_calls = list(dict.fromkeys(tool_calls_seen))[:8]
+        lines.append(f"Tools used: {', '.join(unique_calls)}")
+    if code_snippets:
+        lines.append(f"Code blocks generated: {code_snippets}")
+
+    return "\n".join(lines) if lines else (
+        "Prior conversation turns omitted for context length optimization."
+    )
+
+
+def _cap_summary(text: str, max_tokens: int) -> str:
+    """
+    Truncate summary to stay within max_tokens.
+    Uses tiktoken if available, else len/3.5 heuristic.
+    """
+    try:
+        import tiktoken as _tiktoken
+        enc = _tiktoken.get_encoding("cl100k_base")
+        tokens = enc.encode(text, disallowed_special=())
+        if len(tokens) <= max_tokens:
+            return text
+        # Decode truncated token list back to string
+        return enc.decode(tokens[:max_tokens]) + "..."
+    except Exception:
+        # Fallback: approx 3.5 chars/token
+        char_limit = max_tokens * 3
+        return text[:char_limit] + ("..." if len(text) > char_limit else "")
+
+
+def _summary_token_cap(model: str) -> int:
+    """
+    Max tokens for the <conversation_summary> block, by model family.
+    Codex legacy is very small; GPT-4.x/o-series are generous; large context models can handle more.
+    """
+    ml = model.lower()
+    # Legacy Codex (8k context) — keep summary tiny
+    if any(kw in ml for kw in ("code-davinci", "code-cushman", "codex-001", "codex-002")):
+        return 100
+    # codex-mini / gpt-4o-mini / small models
+    if any(kw in ml for kw in ("codex-mini", "gpt-4o-mini", "gpt-3.5", "o1-mini", "o3-mini")):
+        return 200
+    # Standard GPT-4.x, o1, o3, o4-mini, gpt-5.x
+    if any(kw in ml for kw in ("gpt-4", "gpt-5", "o1", "o3", "o4")):
+        return 350
+    # Large-context models (Claude, Gemini) — be generous
+    return 500
+
+
+def _count_tokens(text: str) -> int:
+    try:
+        import tiktoken as _tiktoken
+        enc = _tiktoken.get_encoding("cl100k_base")
+        return len(enc.encode(text, disallowed_special=()))
+    except Exception:
+        return max(1, int(len(text) / 3.5)) if text else 0
+
+
+def _cap_text_tokens(text: str, max_tokens: int) -> str:
+    if max_tokens <= 0 or _count_tokens(text) <= max_tokens:
+        return text
+    try:
+        import tiktoken as _tiktoken
+        enc = _tiktoken.get_encoding("cl100k_base")
+        tokens = enc.encode(text, disallowed_special=())
+        return enc.decode(tokens[:max_tokens]) + "..."
+    except Exception:
+        return text[:max_tokens * 3] + "..."
+
+
+def _compact_recent_tool_outputs(
+    messages: list[dict],
+    max_recent_tool_tokens: int,
+) -> list[dict]:
+    """
+    Keep recent message structure intact, but cap very large tool/function
+    payloads so retained turns cannot carry raw logs or diffs indefinitely.
+    """
+    if max_recent_tool_tokens <= 0:
+        return messages
+
+    compacted: list[dict] = []
+    for message in messages:
+        role = message.get("role")
+        if role not in ("tool", "function"):
+            compacted.append(message)
+            continue
+
+        content = message.get("content")
+        text = _extract_text(content)
+        if _count_tokens(text) <= max_recent_tool_tokens:
+            compacted.append(message)
+            continue
+
+        capped = _cap_text_tokens(text, max_recent_tool_tokens)
+        next_message = dict(message)
+        next_message["content"] = (
+            "[tool output compacted; original exceeded "
+            f"{max_recent_tool_tokens} tokens]\n{capped}"
+        )
+        compacted.append(next_message)
+
+    return compacted
+
+
+# ── Public API ────────────────────────────────────────────────────────────────
+
+def summarize_openai_history(
+    old_messages: list[dict],
+    model: str = "",
+) -> str:
+    """
+    Produce a compact plain-text summary of old_messages.
+    Used by compact_messages() to build the <conversation_summary> block.
+
+    :param old_messages: Messages being replaced (already sliced — excludes
+                         the system prompt and recent retained turns).
+    :param model:        Target model string (for any model-specific tweaks).
+    :returns: Plain-text summary injected into the summary message.
+    """
+    if not old_messages:
+        return "No prior conversation history."
+    return _summarize_chunk(old_messages)
+
+
+def compact_openai_messages(
+    messages: list[dict],
+    retain_turns: int = 2,
+    model: str = "",
+    max_recent_tool_tokens: int = 800,
+) -> list[dict]:
+    """
+    Compact an OpenAI Chat Completions message array.
+
+    Strategy:
+      1. Separate system / developer messages (always kept first).
+      2. Keep the last `retain_turns` user+assistant exchange pairs intact.
+      3. Replace everything in between with a single summary user message
+         containing a <conversation_summary> XML block.
+      4. For reasoning models (o1/o3/o4) inject summary as a 'user' role
+         (they don't support 'system' injections in all contexts).
+
+    :param messages:      Full message list in OpenAI format.
+    :param retain_turns:  Number of recent exchange pairs to keep.
+    :param model:         Model name for format tuning.
+    :param max_recent_tool_tokens: Cap retained tool/function payloads.
+    :returns: Compacted message list.
+    """
+    if not messages:
+        return messages
+
+    # ── 1. Peel off leading system/developer messages ─────────────────────────────
+    prefix: list[dict] = []
+    body: list[dict] = []
+    in_prefix = True
+    for m in messages:
+        if in_prefix and m.get("role") in ("system", "developer"):
+            prefix.append(m)
+        else:
+            in_prefix = False
+            body.append(m)
+
+    # ── 2. Determine retain window ────────────────────────────────────────────
+    # Each "turn" = one user message + one assistant message (2 items)
+    retain_turns = max(0, int(retain_turns))
+    retain_msgs = retain_turns * 2
+    if len(body) <= retain_msgs:
+        return prefix + _compact_recent_tool_outputs(
+            body, max_recent_tool_tokens=max_recent_tool_tokens)
+
+    if retain_msgs == 0:
+        old_body = body
+        recent_body = []
+    else:
+        old_body = body[:-retain_msgs]
+        recent_body = body[-retain_msgs:]
+
+    # ── 3. Build + cap summary ─────────────────────────────────────────────────
+    raw_summary = summarize_openai_history(old_body, model=model)
+    max_tok = _summary_token_cap(model)
+    summary_text = _cap_summary(raw_summary, max_tok)
+
+    is_reasoning = any(
+        tag in model.lower() for tag in ("o1", "o3", "o4")
+    )
+    # All injections use 'user' role (safe for both chat and reasoning models)
+    _ = is_reasoning  # kept for future per-model tweaks
+
+    summary_message: dict = {
+        "role": "user",
+        "content": (
+            f"<conversation_summary>\n"
+            f"{summary_text}\n"
+            f"</conversation_summary>"
+        ),
+    }
+
+    recent_body = _compact_recent_tool_outputs(
+        recent_body, max_recent_tool_tokens=max_recent_tool_tokens)
+
+    return prefix + [summary_message] + recent_body

package/src/compact/summarizer.py

@@ -0,0 +1,186 @@
+"""
+compact/summarizer.py — Auto-format conversation compaction
+============================================================
+Detects OpenAI / Anthropic / Gemini message format and dispatches
+to the appropriate provider-specific compaction logic.
+"""
+
+from __future__ import annotations
+
+import json
+
+# ── Format detection ──────────────────────────────────────────────────────────
+
+def _detect_format(messages: list[dict]) -> str:
+    """
+    Heuristic format detection.
+
+    Returns: "openai" | "anthropic" | "gemini" | "unknown"
+    """
+    if not messages:
+        return "unknown"
+
+    sample = messages[:5]
+
+    # Gemini: uses 'parts' + 'role' (model | user)
+    for m in sample:
+        if isinstance(m, dict) and "parts" in m:
+            return "gemini"
+
+    # Anthropic: uses 'role' (user | assistant) + possible 'content' list
+    # with 'type' == 'text' | 'image' | 'tool_use' | 'tool_result'
+    for m in sample:
+        if not isinstance(m, dict):
+            continue
+        role = m.get("role", "")
+        content = m.get("content")
+        if role in ("user", "assistant") and isinstance(content, list):
+            for block in content:
+                if isinstance(block, dict) and block.get("type") in (
+                    "text", "image", "tool_use", "tool_result"
+                ):
+                    return "anthropic"
+
+    # OpenAI: uses 'role' with system | developer | tool | function
+    openai_exclusive = {"system", "developer", "tool", "function"}
+    for m in sample:
+        if isinstance(m, dict) and m.get("role") in openai_exclusive:
+            return "openai"
+
+    # Fallback: if messages have 'role' at all, treat as OpenAI (most common)
+    for m in sample:
+        if isinstance(m, dict) and "role" in m:
+            return "openai"
+
+    return "unknown"
+
+
+# ── Provider-specific compaction ──────────────────────────────────────────────
+
+def _compact_openai(
+    messages: list[dict],
+    retain_turns: int,
+    model: str,
+    max_recent_tool_tokens: int,
+) -> list[dict]:
+    from compact.openai import compact_openai_messages
+    return compact_openai_messages(
+        messages,
+        retain_turns=retain_turns,
+        model=model,
+        max_recent_tool_tokens=max_recent_tool_tokens,
+    )
+
+
+def _compact_anthropic(messages: list[dict], retain_turns: int) -> list[dict]:
+    """
+    Compact Anthropic-format messages.
+    Anthropic does not have a 'system' role in the messages array (it's a
+    separate API param), so the body is all user/assistant turns.
+    """
+    from compact.anthropic import summarize_anthropic_history
+    retain_turns = max(0, int(retain_turns))
+    retain_msgs = retain_turns * 2
+    if len(messages) <= retain_msgs:
+        return messages
+
+    if retain_msgs == 0:
+        old_msgs = messages
+        recent = []
+    else:
+        old_msgs = messages[:-retain_msgs]
+        recent = messages[-retain_msgs:]
+
+    summary = summarize_anthropic_history(old_msgs)
+
+    summary_message = {
+        "role": "user",
+        "content": (
+            f"<conversation_summary>\n"
+            f"{summary}\n"
+            f"</conversation_summary>"
+        ),
+    }
+    return [summary_message] + recent
+
+
+def _compact_gemini(messages: list[dict], retain_turns: int) -> list[dict]:
+    """
+    Compact Gemini-format messages (role + parts[]).
+    """
+    from compact.gemini import summarize_gemini_history
+    retain_turns = max(0, int(retain_turns))
+    retain_msgs = retain_turns * 2
+    if len(messages) <= retain_msgs:
+        return messages
+
+    if retain_msgs == 0:
+        old_msgs = messages
+        recent = []
+    else:
+        old_msgs = messages[:-retain_msgs]
+        recent = messages[-retain_msgs:]
+
+    summary = summarize_gemini_history(old_msgs)
+
+    summary_message = {
+        "role": "user",
+        "parts": [
+            {
+                "text": (
+                    f"<conversation_summary>\n"
+                    f"{summary}\n"
+                    f"</conversation_summary>"
+                )
+            }
+        ],
+    }
+    return [summary_message] + recent
+
+
+# ── Public entry-point ────────────────────────────────────────────────────────
+
+def compact_messages(
+    messages_json: str,
+    retain_turns: int = 2,
+    model: str = "",
+    max_recent_tool_tokens: int = 800,
+) -> list[dict]:
+    """
+    Parse messages_json, auto-detect format, and compact in-place.
+
+    :param messages_json: JSON array string of conversation messages.
+    :param retain_turns:  Number of recent turn-pairs (user+assistant) to keep.
+    :param model:         Model hint for provider-specific formatting.
+    :param max_recent_tool_tokens: Cap retained OpenAI tool/function payloads.
+    :returns: Compacted message list.
+    """
+    try:
+        messages: list[dict] = json.loads(messages_json)
+    except (json.JSONDecodeError, ValueError):
+        return []
+
+    if not isinstance(messages, list) or not messages:
+        return []
+
+    fmt = _detect_format(messages)
+
+    if fmt == "openai":
+        return _compact_openai(
+            messages,
+            retain_turns,
+            model=model,
+            max_recent_tool_tokens=max_recent_tool_tokens,
+        )
+    elif fmt == "anthropic":
+        return _compact_anthropic(messages, retain_turns)
+    elif fmt == "gemini":
+        return _compact_gemini(messages, retain_turns)
+    else:
+        # Best-effort: treat as OpenAI
+        return _compact_openai(
+            messages,
+            retain_turns,
+            model=model,
+            max_recent_tool_tokens=max_recent_tool_tokens,
+        )