anolisa-tokenless 0.7.13 → 0.8.0

package/adapters/tokenless/codex/scripts/compress-response

@@ -1,445 +0,0 @@
-#!/usr/bin/env python3
-"""Codex PostToolUse compression hook for the Tokenless plugin.
-
-Reads a PostToolUse hook request from stdin, compresses the tool response
-via ``tokenless compress-response`` and optional TOON encoding, then
-injects a compressed summary and environment error classification as
-``additionalContext``.
-
-Protocol:
-    stdin  ← codex PostToolUse JSON (tool_name, tool_response, ...)
-    stdout → {"hookSpecificOutput": {"hookEventName": "PostToolUse",
-                                      "additionalContext": "..."}}
-
-Design notes for Codex:
-    - PostToolUse CANNOT suppress the original tool output (``suppressOutput``
-      is rejected by the codex hook engine for this event).
-    - ``additionalContext`` is additive — the model sees both the original
-      output AND the injected context.
-    - Therefore we inject a compressed *summary* rather than duplicating the
-      full content, plus environment error classification with fix hints.
-    - For very large JSON responses (> 4 KB), we include the full compressed
-      content so the model has a compact alternative to the raw output.
-
-Fail-open: every error path silently passes through (exit 0 with empty
-stdout) so the agent session is never blocked by a compression failure.
-"""
-
-import json
-import os
-import shutil
-import subprocess
-import sys
-from typing import Optional
-
-# Resolve shared hook utilities (common/hooks/) with FHS fallback paths.
-# Primary: relative path (works for source-tree and FHS/RPM install).
-# Fallbacks: system and user FHS paths (works when Codex caches only the
-# plugin subdirectory and the relative path resolves outside the cache).
-#
-# Trust model (aligned with bash is_trusted_file / Rust is_trusted_path):
-#   - System FHS paths (/usr/share, /usr/local/share) are unconditional.
-#   - User-relative paths use passwd DB home (NOT $HOME — env-controllable),
-#     with ownership and parent-dir permission checks.
-#   - Relative ../../common/hooks is trusted only if the resolved target
-#     exists and is under a system prefix or a passwd-validated home path.
-_HERE = os.path.dirname(os.path.abspath(__file__))
-
-
-def _is_trusted_dir(path: str) -> bool:
-    """Check whether a directory is trusted for Python module imports.
-
-    Mirrors the trust criteria in bash tool_ready_hook.sh (is_trusted_file)
-    and Rust env_check.rs (is_trusted_path): system paths unconditional,
-    user paths validated via passwd + ownership + parent permission checks.
-
-    NOTE: Callers should pass realpath-resolved paths (not just abspath)
-    so that symlink targets are validated before the trust check — a
-    symlinked /usr/share/anolisa → /tmp/ would bypass the prefix check
-    if only abspath is used.
-    """
-    # System FHS prefixes are always trusted
-    for prefix in ("/usr/share/", "/usr/local/share/", "/usr/libexec/", "/usr/lib/anolisa/"):
-        if path.startswith(prefix):
-            return True
-    # For paths outside system prefixes (e.g. source-tree cloned to
-    # /opt/anolisa/), trust them if owned by current uid or root AND
-    # parent directory is not world-writable. This covers developer
-    # worktrees without requiring the path to be under passwd home.
-    try:
-        st = os.stat(path)
-    except OSError:
-        return False
-    if st.st_uid != os.getuid() and st.st_uid != 0:
-        # Not owned by us or root — refuse. Home paths must be owned by
-        # us or root; source-tree paths outside system prefixes are
-        # already covered by the ownership check above (current uid/root).
-        return False
-    # Ownership OK — check parent directory is not world-writable
-    parent = os.path.dirname(path)
-    try:
-        pst = os.stat(parent)
-    except OSError:
-        return False
-    if pst.st_uid != os.getuid() and pst.st_uid != 0:
-        return False
-    if pst.st_mode & 0o002:  # world-writable
-        return False
-    return True
-
-
-# Resolve real home from passwd DB for user-install fallback path.
-# Falls back to "" (empty) when unavailable — candidate is then skipped.
-try:
-    import pwd as _pwd
-    _REAL_HOME = _pwd.getpwuid(os.getuid()).pw_dir
-except (ImportError, KeyError):
-    _REAL_HOME = ""
-if not os.path.isabs(_REAL_HOME):
-    _REAL_HOME = ""
-
-
-def _user_path(*parts: str) -> str:
-    return os.path.join(_REAL_HOME, *parts) if _REAL_HOME else ""
-
-
-_HOOK_UTILS_CANDIDATES = [
-    os.path.join(_HERE, "..", "..", "common", "hooks"),                   # source-tree / FHS
-    "/usr/share/anolisa/adapters/tokenless/common/hooks",                  # RPM system
-    "/usr/local/share/anolisa/adapters/tokenless/common/hooks",            # manual system
-    os.path.join(_REAL_HOME, ".local", "share",
-                 "anolisa", "adapters", "tokenless", "common", "hooks") if _REAL_HOME else "",
-]
-_HOOK_UTILS_RESOLVED = ""
-for _C in _HOOK_UTILS_CANDIDATES:
-    if _C and _is_trusted_dir(os.path.realpath(_C)) and os.path.isdir(_C):
-        _HOOK_UTILS_RESOLVED = os.path.realpath(_C)
-        sys.path.insert(0, _C)
-        break
-
-if _HOOK_UTILS_RESOLVED:
-    from hook_utils import (  # noqa: E402
-        skip_silent as _skip,
-        try_parse_json as _try_parse_json,
-        warn as _warn,
-        classify_env_error as _classify_env_error,
-        get_thresholds as _get_thresholds,
-        SKIP_TOOLS as _SKIP_TOOLS_SHARED,
-    )
-else:
-    # Fail-open: no trusted hook_utils found → provide inline fallbacks
-    # so the hook script exits 0 (passthrough) rather than ImportError → exit 1.
-    def _skip() -> None:  # type: ignore[assignment,misc]
-        sys.exit(0)
-
-    def _try_parse_json(data: str) -> object | None:  # type: ignore[assignment,misc]
-        try:
-            return json.loads(data)
-        except (json.JSONDecodeError, ValueError):
-            return None
-
-    def _warn(msg: str) -> None:  # type: ignore[assignment,misc]
-        print(f"[tokenless] WARNING: {msg}", file=sys.stderr)
-
-    def _classify_env_error(tool_response) -> tuple[str | None, str | None]:  # type: ignore[assignment,misc]
-        # Classification unavailable without hook_utils — return no-op.
-        # Caller handles (None, None) by skipping env error injection.
-        return None, None
-
-    def _get_thresholds(tool_name: str) -> tuple[int, int, int]:  # type: ignore[assignment,misc]
-        # Hardcoded defaults mirroring hook_utils.get_thresholds().
-        # Keep in sync with common/hooks/hook_utils.py thresholds.
-        return (65_536, 128, 8) if tool_name in {"Bash", "bash", "Shell", "shell", "exec", "terminal"} else (1_048_576, 65_536, 32)
-
-    _SKIP_TOOLS_SHARED: set[str] = {  # type: ignore[assignment,misc]
-        "Read", "read", "read_file", "read_many_files",
-        "Glob", "glob", "list_directory",
-        "Grep", "grep", "grep_search", "search_files",
-        "Lsp", "lsp",
-        "NotebookRead", "notebook_read", "notebookread",
-    }
-
-# ---------------------------------------------------------------------------
-# constants
-# ---------------------------------------------------------------------------
-
-AGENT_ID = os.environ.get("TOKENLESS_AGENT_ID", "codex")
-MIN_RESPONSE_CHARS: int = 500
-LARGE_RESPONSE_CHARS: int = 4000
-# TOON on small JSON saves only a few characters (observed ~0.3% below
-# ~500 chars) while the per-event encode cost stays the same, so payloads
-# under this threshold keep the compressed form and skip the TOON pass.
-MIN_TOON_CHARS: int = 500
-
-# 3-layer compression strategy:
-#   Layer 1: Content retrieval + task management → skip all compression
-#   Layer 2: Shell/exec → moderate truncation (64K/128/8)
-#   Layer 3: API/structured → zero-truncation (1M/64K/32)
-
-# Layer 1: content retrieval (shared) + codex-specific task management
-SKIP_TOOLS: set[str] = _SKIP_TOOLS_SHARED | {
-    "TodoWrite", "Task", "TaskStatus",
-}
-
-# Cap for compressed content included in additionalContext (large responses only).
-# Layer 2 thresholds (64K strings) preserve 95% of real shell output; Layer 3
-# zero-truncation (1M strings) can still produce large payloads for API tools.
-COMPRESSED_CONTENT_CAP: int = 32_768
-
-# -- tokenless binary search paths (ANOLISA FHS spec) ------------------------
-# Codex can cache this script without common/hooks. Keep this standalone list
-# aligned with hook_utils.py::_known_binary_paths.
-
-_TOKENLESS_FALLBACK = "/usr/bin/tokenless"
-_TOKENLESS_SYSTEM_FALLBACK = "/usr/local/bin/tokenless"
-_TOKENLESS_LOCAL_BIN = _user_path(".local", "bin", "tokenless")
-_TOKENLESS_LOCAL_SHARE = _user_path(
-    ".local", "share", "anolisa", "tokenless", "tokenless"
-)
-_TOKENLESS_LOCAL_LIB = _user_path(
-    ".local", "lib", "anolisa", "tokenless", "tokenless"
-)
-
-# ---------------------------------------------------------------------------
-# helpers
-# ---------------------------------------------------------------------------
-
-
-def _find_tokenless() -> Optional[str]:
-    """Locate the tokenless CLI binary."""
-    explicit = os.environ.get("TOKENLESS_BIN", "")
-    if explicit and os.path.isfile(explicit) and os.access(explicit, os.X_OK):
-        return explicit
-
-    path = shutil.which("tokenless")
-    if path:
-        return path
-
-    for fp in (
-        _TOKENLESS_LOCAL_BIN,
-        _TOKENLESS_SYSTEM_FALLBACK,
-        _TOKENLESS_FALLBACK,
-        _TOKENLESS_LOCAL_SHARE,
-        _TOKENLESS_LOCAL_LIB,
-    ):
-        if os.path.isfile(fp) and os.access(fp, os.X_OK):
-            return fp
-    return None
-
-
-def _is_json_serializable(obj) -> bool:
-    """Check if a value is a JSON object or array (compressible)."""
-    return isinstance(obj, (dict, list))
-
-
-def _extract_text_for_compression(tool_response) -> Optional[str]:
-    """Extract a JSON string from tool_response for compression."""
-    if isinstance(tool_response, str):
-        return tool_response
-    if isinstance(tool_response, (dict, list)):
-        return json.dumps(tool_response, separators=(",", ":"), ensure_ascii=False)
-    return None
-
-
-def _run_tokenless(tokenless_bin: str, subcommand: str, input_data: str,
-                   session_id: str = "", tool_use_id: str = "",
-                   timeout: int = 10,
-                   extra_args: Optional[list[str]] = None) -> Optional[str]:
-    """Run a tokenless subcommand with input, return stdout or None on failure."""
-    cmd = [tokenless_bin, subcommand, "--agent-id", AGENT_ID]
-    if extra_args:
-        cmd.extend(extra_args)
-    if session_id:
-        cmd.extend(["--session-id", session_id])
-    if tool_use_id:
-        cmd.extend(["--tool-use-id", tool_use_id])
-
-    try:
-        proc = subprocess.run(
-            cmd,
-            input=input_data,
-            capture_output=True,
-            text=True,
-            timeout=timeout,
-        )
-        if proc.returncode == 0 and proc.stdout.strip():
-            return proc.stdout.strip()
-        if proc.returncode != 0 and proc.stderr:
-            _warn(f"{subcommand}: {proc.stderr.strip()[:200]}")
-    except subprocess.TimeoutExpired:
-        _warn(f"{subcommand} timed out after {timeout}s")
-    except OSError as e:
-        _warn(f"{subcommand} OS error: {e}")
-    return None
-
-
-# ---------------------------------------------------------------------------
-# main
-# ---------------------------------------------------------------------------
-
-
-def main() -> None:
-    # 1. Locate tokenless binary (fail-open: skip if not installed)
-    tokenless_bin = _find_tokenless()
-    if not tokenless_bin:
-        _skip()
-
-    # 2. Read and parse stdin JSON from codex
-    try:
-        raw = sys.stdin.read()
-        if not raw.strip():
-            _skip()
-        input_data = json.loads(raw)
-    except (json.JSONDecodeError, EOFError, ValueError):
-        _skip()
-
-    # 3. Extract fields
-    tool_name = input_data.get("tool_name", "")
-    tool_response = input_data.get("tool_response")
-    session_id = input_data.get("session_id", "")
-    tool_use_id = input_data.get("tool_use_id", "")
-
-    # 4. Layer 1: Content retrieval + task management → skip entirely
-    if tool_name in SKIP_TOOLS:
-        _skip()
-
-    # 5. Skip empty or trivial responses
-    if tool_response is None:
-        _skip()
-
-    raw_text = _extract_text_for_compression(tool_response)
-    if not raw_text or raw_text in ("{}", "[]", '""'):
-        _skip()
-
-    # 6. Detect environment errors (always, regardless of size)
-    env_category, env_hint = _classify_env_error(tool_response)
-
-    # 7. Skip compression for small responses, but still report env errors
-    if len(raw_text) < MIN_RESPONSE_CHARS:
-        if env_category:
-            response = {
-                "hookSpecificOutput": {
-                    "hookEventName": "PostToolUse",
-                    "additionalContext": (
-                        f"[tokenless:env:{env_category}] {env_hint} "
-                        f"Do NOT retry the same command — fix the environment first."
-                    ),
-                }
-            }
-            print(json.dumps(response, ensure_ascii=False))
-        _skip()
-
-    # 8. Determine JSON status (needed for compression pipeline)
-    parsed = _try_parse_json(raw_text)
-    is_json = parsed is not None and _is_json_serializable(parsed)
-
-    # 9. Compression pipeline: 3-layer dispatch thresholds
-    #   Layer 1 (content retrieval): already skipped above
-    #   Layer 2 (shell/exec): moderate truncation (64K/128/8)
-    #   Layer 3 (API/structured): zero-truncation (1M/64K/32)
-    compressed_text: Optional[str] = None
-    toon_text: Optional[str] = None
-
-    # -- Build compression input --
-    # If the response is plain text (not JSON), wrap it as a structured
-    # dict so the compression pipeline can still run and record stats.
-    # Otherwise bare strings like git log output would be silently skipped
-    # and never appear in tokenless stats, unlike openclaw/hermes where the
-    # agent framework natively wraps Bash output in {"stdout":...,...}.
-    if is_json:
-        compression_input = raw_text
-    else:
-        compression_input = json.dumps(
-            {"stdout": raw_text}, separators=(",", ":"), ensure_ascii=False
-        )
-
-    # -- Determine thresholds based on tool category --
-    thresholds = _get_thresholds(tool_name)
-
-    # -- Response compression --
-    compressed_text = _run_tokenless(
-        tokenless_bin, "compress-response",
-        compression_input, session_id, tool_use_id, timeout=10,
-        extra_args=[
-            "--truncate-strings-at", str(thresholds[0]),
-            "--truncate-arrays-at", str(thresholds[1]),
-            "--max-depth", str(thresholds[2]),
-        ],
-    )
-    if compressed_text and len(compressed_text) >= len(compression_input):
-        compressed_text = None  # no savings → skip
-
-    # -- TOON encoding (if still valid JSON after compression) --
-    if compressed_text:
-        toon_input = compressed_text
-    else:
-        toon_input = compression_input
-
-    toon_parsed = _try_parse_json(toon_input)
-    if toon_parsed is not None and len(toon_input) >= MIN_TOON_CHARS:
-        toon_text = _run_tokenless(
-            tokenless_bin, "compress-toon",
-            toon_input, session_id, tool_use_id, timeout=10,
-        )
-        if toon_text and len(toon_text) >= len(toon_input):
-            toon_text = None  # no savings → skip
-
-    # 10. Build additionalContext
-    parts: list[str] = []
-
-    original_chars = len(raw_text)
-    final_text = toon_text or compressed_text
-    is_large = len(raw_text) >= LARGE_RESPONSE_CHARS
-
-    if final_text:
-        final_chars = len(final_text)
-        savings_pct = round((1 - final_chars / max(original_chars, 1)) * 100)
-        parts.append(
-            f"[tokenless:compressed] {tool_name}: "
-            f"{original_chars:,} → {final_chars:,} chars ({savings_pct}% reduction)"
-        )
-    else:
-        # No compression savings
-        if is_large:
-            parts.append(
-                f"[tokenless] {tool_name}: {original_chars:,} chars, "
-                f"no compression savings achieved"
-            )
-
-    # Environment error hints
-    if env_category:
-        parts.append(
-            f"[tokenless:env:{env_category}] {env_hint} "
-            f"Do NOT retry the same command — fix the environment first."
-        )
-
-    # For large responses, include the compressed content so the model has a
-    # compact alternative to the raw output.
-    if final_text and is_large:
-        parts.append("--- compressed content ---")
-        if len(final_text) > COMPRESSED_CONTENT_CAP:
-            parts.append(final_text[:COMPRESSED_CONTENT_CAP])
-            parts.append(
-                f"... (compressed content truncated at {COMPRESSED_CONTENT_CAP:,} chars)"
-            )
-        else:
-            parts.append(final_text)
-        parts.append("--- end compressed content ---")
-
-    if not parts:
-        _skip()
-
-    additional_context = "\n".join(parts)
-
-    # 11. Output codex hook response
-    response = {
-        "hookSpecificOutput": {
-            "hookEventName": "PostToolUse",
-            "additionalContext": additional_context,
-        }
-    }
-    print(json.dumps(response, ensure_ascii=False))
-
-
-if __name__ == "__main__":
-    main()

package/adapters/tokenless/common/hooks/compress_toon_hook.py

@@ -1,171 +0,0 @@
-#!/usr/bin/env python3
-"""Tokenless standalone TOON encoding hook.
-
-Reads a PostToolUse JSON from stdin, encodes the tool response
-to TOON format via ``tokenless compress-toon``, and writes a
-HookOutput JSON to stdout.
-
-This is a standalone TOON-only hook for users who want pure TOON
-encoding without response compression.  The combined pipeline
-(response compression + TOON) is in compress_response_hook.py.
-
-Hook point: **PostToolUse**
-
-The agent ID is read from the TOKENLESS_AGENT_ID environment variable
-(set by the install action script).
-"""
-
-import json
-import os
-import subprocess
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-from hook_utils import (
-    _TOKENLESS_FALLBACK,
-    _TOKENLESS_LOCAL_LIB,
-    _TOKENLESS_LOCAL_SHARE,
-    CONTENT_RETRIEVAL_TOOLS,
-    is_skill_file,
-    resolve_agent_id,
-    resolve_binary,
-    skip,
-    try_parse_json,
-    unwrap_string_json,
-    warn,
-)
-
-# -- constants ---------------------------------------------------------------
-
-_AGENT_ID = resolve_agent_id()
-
-# Minimum payload size for TOON encoding. TOON on small JSON saves only a
-# few characters (observed ~0.3% below ~500 chars) while the per-event
-# encode cost stays the same, so smaller responses pass through untouched.
-_MIN_TOON_CHARS = 500
-
-
-# -- main --------------------------------------------------------------------
-
-
-def main() -> None:
-    # 1. Resolve binaries
-    tokenless_bin = resolve_binary(
-        "tokenless",
-        _TOKENLESS_FALLBACK,
-        _TOKENLESS_LOCAL_SHARE,
-        _TOKENLESS_LOCAL_LIB,
-    )
-    if not tokenless_bin:
-        warn("tokenless is not installed. TOON compression hook disabled.")
-        skip()
-
-    # 2. Read stdin JSON
-    try:
-        input_data = json.load(sys.stdin)
-    except (json.JSONDecodeError, EOFError, ValueError):
-        warn("failed to read PostToolUse payload. Passing through unchanged.")
-        skip()
-
-    # 3. Skip content-retrieval tools (preserve integrity)
-    tool_name = input_data.get("tool_name", "unknown")
-    if tool_name in CONTENT_RETRIEVAL_TOOLS:
-        skip()
-
-    # 4. Extract tool_response
-    tool_response_raw = input_data.get("tool_response", "")
-    if not tool_response_raw or tool_response_raw == "{}":
-        skip()
-
-    # 5. Skip skill files (YAML frontmatter)
-    if isinstance(tool_response_raw, str) and is_skill_file(tool_response_raw):
-        skip()
-
-    # 6. Normalize: unwrap string-wrapped JSON
-    if isinstance(tool_response_raw, str):
-        tool_response = unwrap_string_json(tool_response_raw)
-        if tool_response is None:
-            skip()  # Plain text, not JSON
-    elif isinstance(tool_response_raw, (dict, list)):
-        # ensure_ascii=False: the threshold below counts Unicode
-        # characters (code points), not \uXXXX escape sequences, so
-        # structured payloads are measured the same way as JSON string
-        # inputs and the OpenClaw adapter.
-        tool_response = json.dumps(
-            tool_response_raw, separators=(",", ":"), ensure_ascii=False
-        )
-    else:
-        skip()
-
-    if not tool_response:
-        skip()
-
-    # 7. Skip payloads below the TOON minimum threshold (character count,
-    # not byte length): TOON savings on small JSON are near-zero
-    if len(tool_response) < _MIN_TOON_CHARS:
-        skip()
-
-    # 8. Validate it's JSON
-    parsed = try_parse_json(tool_response)
-    if parsed is None:
-        skip()
-
-    # 9. Extract caller context
-    session_id = input_data.get("session_id", "")
-    tool_use_id = input_data.get("tool_use_id") or input_data.get(
-        "toolCallId", ""
-    )
-
-    # 10. Encode to TOON via tokenless compress-toon
-    cmd = [tokenless_bin, "compress-toon", "--agent-id", _AGENT_ID]
-    if session_id:
-        cmd.extend(["--session-id", session_id])
-    if tool_use_id:
-        cmd.extend(["--tool-use-id", tool_use_id])
-
-    try:
-        proc = subprocess.run(
-            cmd,
-            input=tool_response,
-            capture_output=True,
-            text=True,
-            timeout=10,
-        )
-    except Exception as e:
-        warn(f"TOON encoding failed: {e}. Passing through unchanged.")
-        skip()
-
-    if proc.returncode != 0:
-        detail = (proc.stderr or "").strip()[:200]
-        warn(
-            f"TOON encoding exited with code {proc.returncode}: {detail}"
-            if detail
-            else f"TOON encoding exited with code {proc.returncode}. Passing through unchanged."
-        )
-        skip()
-
-    toon_output = proc.stdout.strip()
-    if not toon_output:
-        warn("TOON encoding returned empty output. Passing through unchanged.")
-        skip()
-
-    # 11. Size guard — skip if TOON output is not smaller
-    before_chars = len(tool_response)
-    after_chars = len(toon_output)
-    if after_chars >= before_chars:
-        skip()
-
-    # 12. Build response
-    output = {
-        "suppressOutput": True,
-        "hookSpecificOutput": {
-            "hookEventName": "PostToolUse",
-            "additionalContext": toon_output,
-        },
-    }
-    print(json.dumps(output, ensure_ascii=False))
-
-
-if __name__ == "__main__":
-    main()