anolisa-tokenless 0.7.14 → 0.8.0

package/adapters/tokenless/common/hooks/compress_response_hook.py

@@ -2,15 +2,14 @@
 """Tokenless response compression hook for Cosh-NG, Claude Code, Qoder, and OpenCode.
 
 Reads a PostToolUse JSON from stdin, forwards the model-visible tool
-response to the unified ``tokenless compress`` entry point (protocol v1,
-roadmap §5.4), and translates the CompressionResponse into the host's
+response to the unified ``tokenless compress`` Protocol v2 PostTool operation
+and translates the result into the host's
 envelope. JSON detection, tool threshold selection, TOON selection, and
 final acceptance all live behind the entry point; this hook only parses the
 host object, declares capabilities, and builds envelopes (§4.5).
 
-One Tokenless subprocess per invocation (§5.6). Environment-error
-attribution stays hook-side: it is genuinely additive diagnostics, not a
-compression decision.
+One Tokenless subprocess per invocation. Environment-error attribution is
+owned by the Rust PostTool service.
 
 Hook point: **PostToolUse**
 
@@ -37,10 +36,10 @@ Output contract per agent:
     replacement remain passthrough (roadmap §7). Environment attribution is
     still injected: it is additive by design.
 
-The agent ID is read from the TOKENLESS_AGENT_ID environment variable
-(set by the install action script).  When running under Cosh-NG, the
-agent ID is overridden to ``cosh-ng`` for correct stats attribution.
-Fallback paths follow the ANOLISA FHS spec: /usr/bin/tokenless.
+The agent ID is resolved from the host runtime, ``--agent-id`` argument, or
+TOKENLESS_AGENT_ID environment variable. When running under Cosh-NG, runtime
+detection overrides the declared ID for correct stats attribution. Fallback
+paths follow the ANOLISA FHS spec: /usr/bin/tokenless.
 """
 
 from __future__ import annotations
@@ -56,11 +55,13 @@ from hook_utils import (
     _TOKENLESS_FALLBACK,
     _TOKENLESS_LOCAL_LIB,
     _TOKENLESS_LOCAL_SHARE,
+    SHELL_TOOLS,
     SKIP_TOOLS,
-    build_compression_request,
-    classify_env_error,
+    build_post_tool_request,
+    consume_output_optimization,
     detect_cosh_ng_runtime,
     is_skill_file,
+    is_tokenless_retrieve_command,
     parse_version,
     resolve_agent_id,
     resolve_binary,
@@ -68,17 +69,19 @@ from hook_utils import (
     run_compress,
     secure_write_text,
     skip,
+    tokenless_retrieve_command_available,
     try_parse_json,
     warn,
 )
 
 # -- constants ---------------------------------------------------------------
 
-# Spawn-avoidance mirror of the entry point's 200-char gate. The authority
-# lives in Rust; skipping here only saves the subprocess for content the
-# entry would pass through anyway (normalization never grows the char
-# count, so raw < 200 implies normalized < 200).
-_MIN_RESPONSE_CHARS = 200
+# Shell tool envelopes carry the log in one dominant text field. Unwrapping
+# is worth a rebuilt envelope only when that field is large enough for the
+# build/log engine to bite (its own gates start at 30 lines / 200 chars;
+# 2000 chars keeps the rewrap machinery out of trivial outputs).
+_SHELL_TEXT_FIELDS = ("stdout", "stderr")
+_SHELL_UNWRAP_MIN_CHARS = 2_000
 
 # Below the qwen/cosh extension manifests' 10 s host wrapper so a
 # pathological input is killed here (fail-open skip) before the host kills
@@ -96,9 +99,7 @@ _OPENCODE_AGENT_ID = "opencode"
 # Cache for `claude --version`, keyed on binary path+mtime+size so upgrades
 # invalidate it. Hooks run as a fresh process per tool call and spawning the
 # node CLI every time would add noticeable latency.
-_CLAUDE_VERSION_CACHE = os.path.join(
-    os.path.expanduser("~"), ".tokenless", ".claude-version"
-)
+_CLAUDE_VERSION_CACHE = os.path.join(os.path.expanduser("~"), ".tokenless", ".claude-version")
 
 
 # -- helpers -------------------------------------------------------------------
@@ -115,17 +116,44 @@ def _emit_attribution_or_skip(env_attribution: str) -> None:
     additive and safe on every agent), otherwise a plain skip. Never returns.
     """
     if env_attribution:
-        _emit({
-            "suppressOutput": True,
-            "hookSpecificOutput": {
-                "hookEventName": "PostToolUse",
-                "additionalContext": env_attribution,
-            },
-        })
+        _emit(
+            {
+                "suppressOutput": True,
+                "hookSpecificOutput": {
+                    "hookEventName": "PostToolUse",
+                    "additionalContext": env_attribution,
+                },
+            }
+        )
         sys.exit(0)
     skip()
 
 
+def _shell_text_field(tool_name: str, envelope) -> tuple | None:
+    """The dominant text field of a shell tool's envelope, or ``None``.
+
+    Shell envelopes (``{"stdout": …, "stderr": …}``) are JSON to the entry
+    point, which would compress them log-blind. Unwrapping the largest text
+    field sends the log itself through the text slot; step 13 re-injects the
+    compressed text into a same-shaped envelope, so the host's tool protocol
+    is untouched (adapters own envelope knowledge, §4.5). Only the single
+    largest field is compressed — one Tokenless subprocess per invocation
+    (§5.6) — the other field stays byte-identical.
+    """
+    if tool_name not in SHELL_TOOLS or not isinstance(envelope, dict):
+        return None
+    best = None
+    for name in _SHELL_TEXT_FIELDS:
+        value = envelope.get(name)
+        if (
+            isinstance(value, str)
+            and len(value) >= _SHELL_UNWRAP_MIN_CHARS
+            and (best is None or len(value) > len(best[1]))
+        ):
+            best = (name, value)
+    return best
+
+
 def _cached_claude_version(claude_bin: str) -> tuple | None:
     """Return the Claude Code version tuple, caching `claude --version`."""
     try:
@@ -145,7 +173,9 @@ def _cached_claude_version(claude_bin: str) -> tuple | None:
     try:
         proc = subprocess.run(
             [claude_bin, "--version"],
-            capture_output=True, text=True, timeout=5,
+            capture_output=True,
+            text=True,
+            timeout=5,
         )
     except Exception as e:
         warn(f"claude --version failed: {e}")
@@ -157,9 +187,7 @@ def _cached_claude_version(claude_bin: str) -> tuple | None:
         try:
             # Same hardened write as other ~/.tokenless state files (0o600,
             # symlink-safe) so the cache stays private on shared HOMEs.
-            secure_write_text(
-                _CLAUDE_VERSION_CACHE, f"{cache_key}\n{proc.stdout.strip()}"
-            )
+            secure_write_text(_CLAUDE_VERSION_CACHE, f"{cache_key}\n{proc.stdout.strip()}")
         except OSError:
             pass
     return ver
@@ -187,15 +215,29 @@ def main() -> None:
     cosh_ng_version = detect_cosh_ng_runtime()
     cosh_ng_detected = cosh_ng_version is not None
 
-    # If Cosh-NG is detected but unsupported version, fail open
+    # 2. Resolve agent ID based on runtime
+    agent_id = resolve_agent_id()
+
+    # 3. Read stdin JSON and consume any matching PreTool state.
+    try:
+        input_data = json.load(sys.stdin)
+    except (json.JSONDecodeError, EOFError, ValueError):
+        warn("failed to read PostToolUse payload. Passing through unchanged.")
+        skip()
+
+    session_id = input_data.get("session_id", "")
+    tool_use_id = resolve_tool_call_id(agent_id, input_data)
+    try:
+        output_optimization = consume_output_optimization(agent_id, session_id, tool_use_id)
+    except OSError as error:
+        warn(f"failed to consume PreTool optimization state: {error}")
+        output_optimization = "none"
+
     if cosh_ng_detected and cosh_ng_version == (0, 0, 0):
         warn("Unsupported Cosh-NG version. Response compression disabled (fail open).")
         skip()
 
-    # 2. Resolve agent ID based on runtime
-    agent_id = resolve_agent_id()
-
-    # 3. Resolve binaries
+    # 4. Resolve the single Core entry point after consuming per-call state.
     tokenless_bin = resolve_binary(
         "tokenless", _TOKENLESS_FALLBACK, _TOKENLESS_LOCAL_SHARE, _TOKENLESS_LOCAL_LIB
     )
@@ -203,13 +245,6 @@ def main() -> None:
         warn("tokenless is not installed. Response compression hook disabled.")
         skip()
 
-    # 4. Read stdin JSON
-    try:
-        input_data = json.load(sys.stdin)
-    except (json.JSONDecodeError, EOFError, ValueError):
-        warn("failed to read PostToolUse payload. Passing through unchanged.")
-        skip()
-
     tool_name = input_data.get("tool_name", "unknown")
     tool_response_raw = input_data.get("tool_response", "")
     if not tool_response_raw or tool_response_raw == "{}":
@@ -235,51 +270,36 @@ def main() -> None:
     if isinstance(model_visible_before, str) and is_skill_file(model_visible_before):
         skip()
 
-    # 7. Copy the model-visible value into the request content (§4.5).
-    # ensure_ascii=False matches the entry point's normalization, so size
-    # gates measure Unicode characters on both sides.
-    if isinstance(model_visible_before, str):
+    # 7. Copy the model-visible value into the request content (§4.5). A
+    # shell envelope's dominant text field goes through the text slot
+    # instead of log-blind JSON; ensure_ascii=False matches the entry
+    # point's normalization, so size gates measure Unicode characters on
+    # both sides.
+    shell_field = _shell_text_field(tool_name, model_visible_before)
+    if shell_field is not None:
+        content = shell_field[1]
+    elif isinstance(model_visible_before, str):
         content = model_visible_before
     elif isinstance(model_visible_before, (dict, list)):
-        content = json.dumps(
-            model_visible_before, separators=(",", ":"), ensure_ascii=False
-        )
+        content = json.dumps(model_visible_before, separators=(",", ":"), ensure_ascii=False)
     else:
         skip()
 
-    # 8. Extract caller context
-    session_id = input_data.get("session_id", "")
-    tool_use_id = resolve_tool_call_id(agent_id, input_data)
-
-    # 9. Environment attribution analysis — additive diagnostics, computed
-    # hook-side. Only structured payloads are classified (with the same
-    # string-unwrap the entry point applies): plain text never reached
-    # attribution in the two-subprocess hook and still does not.
-    if isinstance(model_visible_before, dict):
-        attr_subject = model_visible_before
-    else:
-        parsed = try_parse_json(content)
-        if isinstance(parsed, str):
-            parsed = try_parse_json(parsed)
-        attr_subject = parsed if isinstance(parsed, (dict, list)) else None
-    env_attribution = ""
-    attr_category, attr_fix_hint = classify_env_error(attr_subject)
-    if attr_category:
-        env_attribution = (
-            f"[tokenless:env] {tool_name} failed: "
-            f"{attr_category} ({attr_fix_hint}). Skip retry."
-        )
-
-    # 10. Capability declaration (§4.5): what can this host actually do?
+    # 8. Capability declaration: what can this host actually do?
     if cosh_ng_detected:
         can_replace = True
         replace_with_text = True  # updatedToolResponse accepts any text
     elif agent_id in {_QODER_AGENT_ID, _OPENCODE_AGENT_ID}:
         can_replace = True
-        replace_with_text = not isinstance(tool_response_raw, (dict, list))
+        # An unwrapped shell field is plain text regardless of its envelope.
+        replace_with_text = shell_field is not None or not isinstance(
+            tool_response_raw, (dict, list)
+        )
     elif agent_id == _CLAUDE_AGENT_ID:
         can_replace = _claude_supports_replacement()
-        replace_with_text = not isinstance(tool_response_raw, (dict, list))
+        replace_with_text = shell_field is not None or not isinstance(
+            tool_response_raw, (dict, list)
+        )
         if not can_replace:
             warn(
                 "Claude Code < 2.1.121 (or version unknown): "
@@ -291,30 +311,81 @@ def main() -> None:
         can_replace = False
         replace_with_text = True
 
-    # 11. Spawn-avoidance prefilters. The entry point re-checks all three
-    # authoritatively; skipping here just saves the exec. SKIP_TOOLS reads
-    # the same tool_categories.json the entry point embeds, so content
-    # retrieval — the hottest PostToolUse traffic — never pays a spawn.
-    if not can_replace:
-        _emit_attribution_or_skip(env_attribution)
+    # 9. Map host facts into the required lifecycle fields.
     if tool_name in SKIP_TOOLS:
-        _emit_attribution_or_skip(env_attribution)
-    if len(content) < _MIN_RESPONSE_CHARS:
-        _emit_attribution_or_skip(env_attribution)
+        content_origin = "file_content"
+    elif tool_name in SHELL_TOOLS:
+        content_origin = "command_output"
+    else:
+        content_origin = "api_response"
+    raw_status = str(input_data.get("status", "")).lower()
+    shell_process_result = model_visible_before if isinstance(model_visible_before, dict) else None
+    shell_process_error = (
+        tool_name in SHELL_TOOLS
+        and shell_process_result is not None
+        and (
+            shell_process_result.get("error") is not None
+            or (
+                shell_process_result.get("exit_code") is not None
+                and shell_process_result.get("exit_code") != 0
+            )
+            or (
+                shell_process_result.get("exitCode") is not None
+                and shell_process_result.get("exitCode") != 0
+            )
+        )
+    )
+    if raw_status in {"interrupted", "denied"}:
+        status = raw_status
+    elif input_data.get("is_error") is True or (
+        isinstance(tool_response_raw, dict) and tool_response_raw.get("isError") is True
+    ):
+        status = "error"
+    elif shell_process_error:
+        status = "error"
+    else:
+        status = "success"
+
+    # Shell envelopes often carry a large stdout alongside the actual failure
+    # in a short stderr. Error results are never replaced, so send the error
+    # stream to Core for diagnosis while the host keeps the original envelope.
+    if status == "error" and tool_name in SHELL_TOOLS and isinstance(model_visible_before, dict):
+        error_parts = []
+        for field in ("stderr", "error"):
+            value = model_visible_before.get(field)
+            if isinstance(value, str) and value.strip():
+                error_parts.append(value)
+        if error_parts:
+            content = "\n".join(error_parts)
+
+    retrieve_result = status == "success" and is_tokenless_retrieve_command(
+        tool_name, input_data.get("tool_input")
+    )
+    retrieval_available = (
+        can_replace
+        and status == "success"
+        and output_optimization == "none"
+        and not retrieve_result
+        and tokenless_retrieve_command_available()
+    )
 
-    # 12. The one Tokenless subprocess: the unified entry point decides.
-    request = build_compression_request(
+    # 10. The one Tokenless subprocess: Core owns all PostTool policy.
+    request = build_post_tool_request(
         content,
         agent_id,
-        "post_tool",
+        tool_name,
+        status,
+        content_origin,
+        output_optimization,
+        result_kind="retrieve" if retrieve_result else "tool",
+        recovery={"kind": "shell" if retrieval_available else "none"},
         session_id=session_id,
         tool_use_id=tool_use_id,
-        tool_name=tool_name,
-        replace_output=True,
-        publish_retrieve_tool=True,
+        replace_output=can_replace,
         replace_with_text=replace_with_text,
     )
-    response = run_compress(tokenless_bin, request, _COMPRESS_TIMEOUT)
+    response = run_compress(tokenless_bin, request, _COMPRESS_TIMEOUT, "post_tool")
+    env_attribution = response.get("additional_context", "") if response is not None else ""
     if response is None or response.get("disposition") != "applied":
         _emit_attribution_or_skip(env_attribution)
 
@@ -323,18 +394,28 @@ def main() -> None:
         warn("tokenless compress returned no output. Passing through unchanged.")
         _emit_attribution_or_skip(env_attribution)
 
-    # 13. Envelope construction — dispatch by agent runtime.
+    # 11. Envelope construction — dispatch by agent runtime. An unwrapped
+    # shell field is re-injected into a same-shaped envelope: the compressed
+    # text replaces exactly the field that was sent, every other field stays
+    # byte-identical.
+    rewrapped = None
+    if shell_field is not None:
+        rewrapped = dict(model_visible_before)
+        rewrapped[shell_field[0]] = output_text
+
     if cosh_ng_detected:
         hook_specific = {
             "hookEventName": "PostToolUse",
-            "updatedToolResponse": output_text,
+            "updatedToolResponse": rewrapped if rewrapped is not None else output_text,
         }
         if env_attribution:
             hook_specific["additionalContext"] = env_attribution
         _emit({"suppressOutput": True, "hookSpecificOutput": hook_specific})
         return
 
-    if replace_with_text:
+    if rewrapped is not None:
+        updated_output = rewrapped
+    elif replace_with_text:
         updated_output = output_text
     else:
         # Structured slot: the entry point guarantees schema-stable JSON for
@@ -347,9 +428,12 @@ def main() -> None:
 
     # Qoder validates updatedToolOutput as a string even when the original
     # tool response is structured. The entry point's compact serialization
-    # is exactly that string.
+    # is exactly that string; a rewrapped shell envelope serializes here.
     if agent_id == _QODER_AGENT_ID and not isinstance(updated_output, str):
-        updated_output = output_text
+        if rewrapped is not None:
+            updated_output = json.dumps(rewrapped, separators=(",", ":"), ensure_ascii=False)
+        else:
+            updated_output = output_text
 
     hook_output = {
         "hookEventName": "PostToolUse",

package/adapters/tokenless/common/hooks/compress_schema_hook.py

@@ -2,8 +2,8 @@
 """Tokenless schema compression hook.
 
 Reads a BeforeModel JSON from stdin, extracts the tools array, forwards it
-to the unified ``tokenless compress`` entry point (protocol v1, seam
-``before_model``, roadmap §5.4), and writes a HookOutput JSON to stdout.
+to the unified ``tokenless compress`` Protocol v2 BeforeModel operation and
+writes a HookOutput JSON to stdout.
 The entry point returns the original array on no-savings, which this hook
 wraps exactly like a compressed one — the historical behavior of the
 ``compress-schema`` flow it replaces.
@@ -32,7 +32,7 @@ from hook_utils import (
     _TOKENLESS_FALLBACK,
     _TOKENLESS_LOCAL_LIB,
     _TOKENLESS_LOCAL_SHARE,
-    build_compression_request,
+    build_before_model_request,
     resolve_agent_id,
     resolve_binary,
     resolve_tool_call_id,
@@ -112,14 +112,6 @@ def _marker_lock():
 # -- helpers -----------------------------------------------------------------
 
 
-def _is_json_array(data: str) -> bool:
-    try:
-        obj = json.loads(data)
-        return isinstance(obj, list)
-    except (json.JSONDecodeError, ValueError):
-        return False
-
-
 def _session_warn_key(session_id: str) -> str:
     """Normalize the session ID into the dedup key for the no-tools warning.
 
@@ -275,37 +267,33 @@ def main() -> None:
                 )
         skip()
 
-    tools_json = json.dumps(tools, separators=(",", ":"))
-
     # 4. Extract caller context
     session_id = input_data.get("session_id", "")
     tool_use_id = resolve_tool_call_id(_AGENT_ID, input_data)
 
     # 5. Compress schemas via the unified entry point (one subprocess).
-    request = build_compression_request(
-        tools_json,
+    llm_request = input_data.get("llm_request", {})
+    visible_context = dict(llm_request) if isinstance(llm_request, dict) else {}
+    visible_context.pop("tools", None)
+    config = visible_context.get("config")
+    if isinstance(config, dict):
+        visible_context["config"] = dict(config)
+        visible_context["config"].pop("tools", None)
+    request = build_before_model_request(
+        tools,
+        visible_context,
         _AGENT_ID,
-        "before_model",
         session_id=session_id,
         tool_use_id=tool_use_id,
-        replace_output=True,
-        publish_retrieve_tool=True,
     )
-    response = run_compress(tokenless_bin, request, _COMPRESS_TIMEOUT)
+    response = run_compress(
+        tokenless_bin, request, _COMPRESS_TIMEOUT, "before_model"
+    )
     if response is None:
         warn("Schema compression subprocess failed. Passing through unchanged.")
         skip()
-    if response.get("disposition") in {"error", "timeout"}:
-        warn("Schema compression failed. Passing through unchanged.")
-        skip()
-    if response.get("disposition") == "reversibility_unavailable":
-        # Savings existed but the stash could not record the originals;
-        # the entry point returned the uncompressed schemas. Surface the
-        # distinction from "nothing to compress" (envelope unchanged).
-        warn("Schema stash unavailable; truncated descriptions would be lossy.")
-
-    compressed = response.get("output")
-    if not isinstance(compressed, str) or not _is_json_array(compressed):
+    compressed = response.get("tools")
+    if not isinstance(compressed, list):
         warn(
             "Schema compression returned invalid JSON. Passing through unchanged."
         )
@@ -317,7 +305,7 @@ def main() -> None:
             "hookEventName": "BeforeModel",
             "llm_request": {
                 "config": {
-                    "tools": json.loads(compressed),
+                    "tools": compressed,
                 },
             },
         },