ctxora 6.2.0

package/src/chunking/compressor.py

@@ -0,0 +1,104 @@
+"""
+chunking/compressor.py — Context compressor
+============================================
+Compresses chunks into a compact representation depending on the target
+model provider:
+  - Claude  → XML <context> blocks
+  - OpenAI  → Markdown comments (minimal token overhead)
+  - Codex   → Single-line summary + truncated code (max 30 lines)
+  - Gemini  → JSON object
+"""
+
+from __future__ import annotations
+
+import json
+
+
+class ContextCompressor:
+    def __init__(self) -> None:
+        pass
+
+    def compress(
+        self,
+        chunks: list,
+        model: str = "",
+    ) -> list[str]:
+        """
+        Compress each chunk to its minimal useful representation.
+
+        :param chunks: List of Chunk objects.
+        :param model:  Target model string for format selection.
+        :returns: List of compressed strings (one per chunk).
+        """
+        ml = model.lower()
+
+        if "gemini" in ml:
+            return [self._compress_gemini(c) for c in chunks]
+        if any(kw in ml for kw in ("codex", "code-davinci", "code-cushman")):
+            return [self._compress_codex(c) for c in chunks]
+        if "claude" in ml or "anthropic" in ml:
+            return [self._compress_claude(c) for c in chunks]
+        # Default: OpenAI markdown (also used for gpt-*, o1/o3/o4)
+        return [self._compress_openai(c) for c in chunks]
+
+    # ── Claude XML ────────────────────────────────────────────────────────────
+    def _compress_claude(self, c) -> str:
+        summary = c.summary if c.summary else f"{c.type} {c.symbol}"
+        critical = "\n".join(c.content.splitlines()[:60])
+        return (
+            f"<context>\n"
+            f"  <summary>{summary}</summary>\n"
+            f"  <critical_code>\n{critical}\n  </critical_code>\n"
+            f"</context>"
+        )
+
+    # ── OpenAI Markdown ───────────────────────────────────────────────────────
+    def _compress_openai(self, c) -> str:
+        summary = c.summary if c.summary else f"{c.type} `{c.symbol}`"
+        critical = "\n".join(c.content.splitlines()[:60])
+        lang = _ext_to_lang(c.path)
+        return (
+            f"<!-- {c.path} | {summary} -->\n"
+            f"```{lang}\n{critical}\n```"
+        )
+
+    # ── Codex — ultra-compact ─────────────────────────────────────────────────
+    def _compress_codex(self, c) -> str:
+        summary = c.summary if c.summary else f"{c.type} {c.symbol}"
+        # Only 30 lines for legacy Codex (tiny context)
+        critical = "\n".join(c.content.splitlines()[:30])
+        lang = _ext_to_lang(c.path)
+        return (
+            f"// {c.path} [{summary}]\n"
+            f"```{lang}\n{critical}\n```"
+        )
+
+    # ── Gemini JSON ───────────────────────────────────────────────────────────
+    def _compress_gemini(self, c) -> str:
+        return json.dumps({
+            "path": c.path,
+            "symbol": c.symbol,
+            "summary": c.summary or f"{c.type} {c.symbol}",
+            "snippet": "\n".join(c.content.splitlines()[:60]),
+        }, ensure_ascii=False)
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+_EXT_LANG: dict[str, str] = {
+    ".py": "python", ".ts": "typescript", ".tsx": "tsx",
+    ".js": "javascript", ".jsx": "jsx", ".go": "go",
+    ".rs": "rust", ".java": "java", ".cs": "csharp",
+    ".cpp": "cpp", ".c": "c", ".h": "c",
+    ".rb": "ruby", ".php": "php", ".swift": "swift",
+    ".sh": "bash", ".zsh": "bash", ".md": "markdown",
+    ".json": "json", ".yaml": "yaml", ".yml": "yaml",
+    ".toml": "toml", ".html": "html", ".css": "css", ".sql": "sql",
+}
+
+
+def _ext_to_lang(path: str) -> str:
+    if not path:
+        return ""
+    dot = path.rfind(".")
+    return _EXT_LANG.get(path[dot:].lower(), "") if dot != -1 else ""

package/src/chunking/treesitter_chunker.py

@@ -0,0 +1,240 @@
+"""
+chunking/treesitter_chunker.py — AST-based code chunker
+=========================================================
+Uses tree-sitter to split files into function/class-level chunks.
+Falls back to file-level chunking for unsupported languages.
+
+Token counting:
+  - tiktoken (cl100k_base) for accurate GPT/Codex token counts
+  - Falls back to len//3.5 heuristic if tiktoken unavailable
+"""
+
+from __future__ import annotations
+
+import ast
+import hashlib
+import os
+from dataclasses import dataclass, field
+
+# ── Token counting ────────────────────────────────────────────────────────────
+
+try:
+    import tiktoken as _tiktoken
+    # cl100k_base covers: gpt-4, gpt-3.5-turbo, gpt-4o, codex-mini, o1/o3/o4
+    # o200k_base covers: gpt-4o (newer), but cl100k_base is ≥95% accurate for both
+    _ENCODER = _tiktoken.get_encoding("cl100k_base")
+    _HAS_TIKTOKEN = True
+except Exception:
+    _HAS_TIKTOKEN = False
+    _ENCODER = None  # type: ignore[assignment]
+
+
+def count_tokens(text: str) -> int:
+    """
+    Count tokens in text.
+
+    Uses tiktoken (cl100k_base) for OpenAI-compatible accuracy.
+    Falls back to len/3.5 heuristic if tiktoken is unavailable.
+
+    cl100k_base is accurate for:
+      - GPT-4, GPT-4o, GPT-4.1, GPT-4.5, GPT-5.x
+      - GPT-3.5-turbo
+      - o1, o3, o4-mini
+      - codex-mini-latest
+      - Claude (close approximation, actual tokenizer differs ~5%)
+      - Gemini (close approximation)
+    """
+    if not text:
+        return 0
+    if _HAS_TIKTOKEN and _ENCODER is not None:
+        return len(_ENCODER.encode(text, disallowed_special=()))
+    # Fallback: code typically has ratio 3.2–3.8 chars/token; use 3.5
+    return max(1, int(len(text) / 3.5))
+
+
+# ── Chunk dataclass ───────────────────────────────────────────────────────────
+
+@dataclass
+class Chunk:
+    id: str
+    path: str
+    type: str
+    symbol: str
+    content: str
+    summary: str
+    tokens: int
+    embedding: list[float] = field(default_factory=list)
+    imports: list[str] = field(default_factory=list)
+    exports: list[str] = field(default_factory=list)
+    priority: int = 5
+    workspace_id: str = "legacy_global"
+    start_line: int = 1
+    end_line: int = 1
+    content_hash: str = ""
+
+    @staticmethod
+    def stable_id(path: str, kind: str, symbol: str, start_line: int, end_line: int, content: str) -> str:
+        payload = f"{path}\0{kind}\0{symbol}\0{start_line}\0{end_line}\0{hashlib.sha256(content.encode('utf-8')).hexdigest()}"
+        return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+# ── AST Chunker ───────────────────────────────────────────────────────────────
+
+class ASTChunker:
+    def __init__(self):
+        try:
+            import tree_sitter
+            import tree_sitter_javascript
+            import tree_sitter_python
+            import tree_sitter_typescript
+            self.ts = tree_sitter
+            self.lang_py = tree_sitter.Language(tree_sitter_python.language())
+            self.lang_js = tree_sitter.Language(
+                tree_sitter_javascript.language())
+            self.lang_ts = tree_sitter.Language(
+                tree_sitter_typescript.language_typescript())
+        except ImportError:
+            self.ts = None
+
+    def get_parser(self, ext: str):
+        if not self.ts:
+            return None
+        parser = self.ts.Parser()
+        if ext == ".py":
+            parser.language = self.lang_py
+        elif ext in [".js", ".jsx"]:
+            parser.language = self.lang_js
+        elif ext in [".ts", ".tsx"]:
+            parser.language = self.lang_ts
+        else:
+            return None
+        return parser
+
+    def chunk_paths(self, paths: list[str]) -> list[Chunk]:
+        chunks = []
+        for path in paths:
+            if not os.path.exists(path):
+                continue
+            if os.path.isdir(path):
+                for root, _, files in os.walk(path):
+                    for file in files:
+                        filepath = os.path.join(root, file)
+                        chunks.extend(self._chunk_file(filepath))
+            else:
+                chunks.extend(self._chunk_file(path))
+        return chunks
+
+    def _chunk_file(self, filepath: str) -> list[Chunk]:
+        chunks = []
+        ext = os.path.splitext(filepath)[1]
+        try:
+            with open(filepath, "r", encoding="utf-8") as f:
+                content = f.read()
+        except Exception:
+            return []
+
+        if ext == ".py":
+            try:
+                tree = ast.parse(content)
+            except SyntaxError:
+                tree = None
+            if tree is not None:
+                lines = content.splitlines()
+                import_nodes = [node for node in tree.body if isinstance(node, (ast.Import, ast.ImportFrom))]
+                import_end = max((node.end_lineno or node.lineno for node in import_nodes), default=0)
+                import_context = "\n".join(lines[:import_end])
+                python_chunks = []
+                for node in tree.body:
+                    if not isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+                        continue
+                    start_line = min([node.lineno] + [decorator.lineno for decorator in node.decorator_list])
+                    end_line = node.end_lineno or start_line
+                    body = "\n".join(lines[start_line - 1:end_line])
+                    chunk_content = f"{import_context}\n\n{body}" if import_context else body
+                    kind = "class_definition" if isinstance(node, ast.ClassDef) else "function_definition"
+                    python_chunks.append(Chunk(
+                        id=Chunk.stable_id(filepath, kind, node.name, start_line, end_line, chunk_content),
+                        path=filepath, type=kind, symbol=node.name, content=chunk_content,
+                        summary=f"{kind} {node.name}", tokens=count_tokens(chunk_content),
+                        start_line=start_line, end_line=end_line,
+                        content_hash=hashlib.sha256(chunk_content.encode("utf-8")).hexdigest(),
+                    ))
+                if python_chunks:
+                    return python_chunks
+
+        parser = self.get_parser(ext)
+        if parser is None:
+            # Fallback to file-level chunk
+            return [
+                Chunk(
+                    id=Chunk.stable_id(filepath, "file", "module", 1, len(content.splitlines()), content),
+                    path=filepath,
+                    type="file",
+                    symbol="module",
+                    content=content,
+                    summary="",
+                    tokens=count_tokens(content),
+                    start_line=1,
+                    end_line=max(1, len(content.splitlines())),
+                    content_hash=hashlib.sha256(content.encode("utf-8")).hexdigest(),
+                )
+            ]
+
+        tree = parser.parse(bytes(content, "utf8"))
+        root_node = tree.root_node
+        content_bytes = bytes(content, "utf8")
+
+        def traverse(node):
+            if node.type in [
+                "class_definition",
+                "function_definition",
+                "class_declaration",
+                "function_declaration",
+                "method_definition",
+            ]:
+                chunk_content = content_bytes[
+                    node.start_byte:node.end_byte
+                ].decode("utf-8")
+                symbol_name = "unknown"
+                for child in node.children:
+                    if child.type == "identifier":
+                        symbol_name = content_bytes[
+                            child.start_byte:child.end_byte
+                        ].decode("utf-8")
+                        break
+
+                start_line = node.start_point[0] + 1
+                end_line = node.end_point[0] + 1
+                chunks.append(Chunk(
+                    id=Chunk.stable_id(filepath, node.type, symbol_name, start_line, end_line, chunk_content),
+                    path=filepath,
+                    type=node.type,
+                    symbol=symbol_name,
+                    content=chunk_content,
+                    summary=f"{node.type} {symbol_name}",
+                    tokens=count_tokens(chunk_content),
+                    start_line=start_line,
+                    end_line=end_line,
+                    content_hash=hashlib.sha256(chunk_content.encode("utf-8")).hexdigest(),
+                ))
+            for child in node.children:
+                traverse(child)
+
+        traverse(root_node)
+
+        # If no chunks extracted, add file as a module chunk
+        if not chunks:
+            chunks.append(Chunk(
+                id=Chunk.stable_id(filepath, "module", "module", 1, len(content.splitlines()), content),
+                path=filepath,
+                type="module",
+                symbol="module",
+                content=content,
+                summary="",
+                tokens=count_tokens(content),
+                start_line=1,
+                end_line=max(1, len(content.splitlines())),
+                content_hash=hashlib.sha256(content.encode("utf-8")).hexdigest(),
+            ))
+
+        return chunks

package/src/compact/anthropic.py

@@ -0,0 +1,98 @@
+"""
+compact/anthropic.py — Anthropic-specific format helpers
+=========================================================
+Provides a standalone function for summarizing Anthropic-format
+old messages. The main compaction entry-point is in summarizer.py.
+
+Anthropic content blocks may be:
+  - str
+  - list[dict] with type: "text" | "image" | "tool_use" | "tool_result"
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def _extract_text(content: Any) -> str:
+    """Flatten Anthropic content to plain text."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            t = block.get("type", "")
+            if t == "text":
+                parts.append(block.get("text", ""))
+            elif t == "image":
+                parts.append("[image]")
+            elif t == "tool_use":
+                parts.append(f"[tool: {block.get('name', '')}]")
+            elif t == "tool_result":
+                inner = block.get("content", "")
+                if isinstance(inner, str):
+                    parts.append(f"[tool_result: {inner[:80]}]")
+        return " ".join(parts)
+    return str(content) if content is not None else ""
+
+
+def summarize_anthropic_history(
+    old_messages: list[dict],
+    model: str = "",
+) -> str:
+    """
+    Produce a compact plain-text summary of old_messages.
+
+    :param old_messages: Messages being replaced (excludes retained turns).
+    :param model:        Target Claude model string (for future tuning).
+    :returns: Plain-text summary injected into the summary message.
+    """
+    if not old_messages:
+        return "No prior conversation history."
+
+    user_intents: list[str] = []
+    assistant_actions: list[str] = []
+    tool_calls_seen: list[str] = []
+
+    for m in old_messages:
+        role = m.get("role", "")
+        text = _extract_text(m.get("content") or "").strip()
+
+        if role == "user" and text:
+            first = re.split(r"[.!?\n]", text)[0].strip()
+            if first:
+                user_intents.append(first[:120])
+
+        elif role == "assistant":
+            content = m.get("content", [])
+            if isinstance(content, list):
+                for block in content:
+                    if isinstance(block, dict):
+                        if block.get("type") == "tool_use":
+                            tool_calls_seen.append(block.get("name", ""))
+                        elif block.get("type") == "text":
+                            t = block.get("text", "").strip()
+                            if t:
+                                first = re.split(r"[.!?\n]", t)[0].strip()
+                                if first:
+                                    assistant_actions.append(first[:120])
+            elif text:
+                first = re.split(r"[.!?\n]", text)[0].strip()
+                if first:
+                    assistant_actions.append(first[:120])
+
+    lines: list[str] = []
+    if user_intents:
+        lines.append(f"User requests: {'; '.join(user_intents[:5])}")
+    if assistant_actions:
+        lines.append(f"Assistant actions: {'; '.join(assistant_actions[:5])}")
+    if tool_calls_seen:
+        unique = list(dict.fromkeys(tool_calls_seen))[:8]
+        lines.append(f"Tools used: {', '.join(unique)}")
+
+    return "\n".join(lines) if lines else (
+        "Prior conversation turns omitted for context length optimization (Anthropic)."
+    )

package/src/compact/gemini.py

@@ -0,0 +1,88 @@
+"""
+compact/gemini.py — Gemini-specific format helpers
+===================================================
+Gemini uses role + parts[] instead of role + content.
+Parts can be: text str, inline_data (images), function_call, function_response.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def _extract_text_from_parts(parts: Any) -> str:
+    """Flatten Gemini parts array to plain text."""
+    if not isinstance(parts, list):
+        return str(parts) if parts else ""
+    texts: list[str] = []
+    for p in parts:
+        if isinstance(p, str):
+            texts.append(p)
+        elif isinstance(p, dict):
+            if "text" in p:
+                texts.append(p["text"])
+            elif p.get("inline_data"):
+                texts.append("[inline_data]")
+            elif p.get("function_call"):
+                fn = p["function_call"].get("name", "")
+                texts.append(f"[function_call: {fn}]")
+            elif p.get("function_response"):
+                fn = p["function_response"].get("name", "")
+                texts.append(f"[function_response: {fn}]")
+    return " ".join(texts)
+
+
+def summarize_gemini_history(
+    old_messages: list[dict],
+    model: str = "",
+) -> str:
+    """
+    Produce a compact plain-text summary of old_messages in Gemini format.
+
+    :param old_messages: Messages being replaced (excludes retained turns).
+    :param model:        Target Gemini model string.
+    :returns: Plain-text summary.
+    """
+    if not old_messages:
+        return "No prior conversation history."
+
+    user_intents: list[str] = []
+    model_actions: list[str] = []
+    fn_calls_seen: list[str] = []
+
+    for m in old_messages:
+        role = m.get("role", "")  # "user" | "model"
+        parts = m.get("parts", [])
+
+        # Collect function calls
+        for p in parts if isinstance(parts, list) else []:
+            if isinstance(p, dict):
+                if p.get("function_call"):
+                    fn = p["function_call"].get("name", "")
+                    if fn:
+                        fn_calls_seen.append(fn)
+
+        text = _extract_text_from_parts(parts).strip()
+
+        if role == "user" and text:
+            first = re.split(r"[.!?\n]", text)[0].strip()
+            if first:
+                user_intents.append(first[:120])
+        elif role == "model" and text:
+            first = re.split(r"[.!?\n]", text)[0].strip()
+            if first:
+                model_actions.append(first[:120])
+
+    lines: list[str] = []
+    if user_intents:
+        lines.append(f"User requests: {'; '.join(user_intents[:5])}")
+    if model_actions:
+        lines.append(f"Model actions: {'; '.join(model_actions[:5])}")
+    if fn_calls_seen:
+        unique = list(dict.fromkeys(fn_calls_seen))[:8]
+        lines.append(f"Functions called: {', '.join(unique)}")
+
+    return "\n".join(lines) if lines else (
+        "Prior conversation turns omitted for context length optimization (Gemini)."
+    )