anolisa-tokenless 0.7.13 → 0.8.0

package/adapters/tokenless/openclaw/openclaw.plugin.json

@@ -1,8 +1,8 @@
 {
   "id": "tokenless",
   "name": "Tokenless",
-  "version": "0.7.13",
-  "description": "Unified RTK command rewriting + response/TOON compression + registered but hard-disabled Tool Ready. Wraps tokenless and rtk system binaries via child_process — this is expected and not malicious.",
+  "version": "0.8.0",
+  "description": "Protocol v2 PreTool RTK rewriting and PostTool optimization. Wraps the tokenless system binary via child_process — this is expected and not malicious.",
   "activation": {
     "onCapabilities": ["hook"]
   },
@@ -11,30 +11,14 @@
     "properties": {
       "rtk_enabled": { "type": "boolean", "default": true },
       "tool_ready_enabled": { "type": "boolean", "default": true },
-      "response_compression_enabled": { "type": "boolean", "default": true },
-      "skip_tools": {
-        "type": "array",
-        "items": { "type": "string" },
-        "default": [],
-        "description": "Layer 1: Tools that skip all compression (preserve integrity). Loaded from tool_categories.json if not specified."
-      },
-      "shell_tools": {
-        "type": "array",
-        "items": { "type": "string" },
-        "default": [],
-        "description": "Layer 2: Shell/exec tools with moderate truncation. Loaded from tool_categories.json if not specified."
-      },
-      "toon_compression_enabled": { "type": "boolean", "default": false },
+      "post_tool_enabled": { "type": "boolean", "default": true },
       "verbose": { "type": "boolean", "default": false }
     }
   },
   "uiHints": {
     "rtk_enabled": { "label": "Enable RTK command rewriting" },
     "tool_ready_enabled": { "label": "Register Tool Ready hook (hard-disabled)" },
-    "response_compression_enabled": { "label": "Enable response compression" },
-    "skip_tools": { "label": "Tool names whose responses should not be compressed" },
-    "shell_tools": { "label": "Tool names treated as shell/exec (moderate truncation)" },
-    "toon_compression_enabled": { "label": "Enable TOON format compression" },
+    "post_tool_enabled": { "label": "Enable PostTool optimization" },
     "verbose": { "label": "Verbose logging" }
   }
 }

package/adapters/tokenless/openclaw/package.json

@@ -1,18 +1,20 @@
 {
   "name": "@tokenless/openclaw-plugin",
-  "version": "0.7.13",
-  "description": "Unified OpenClaw plugin — RTK command rewriting + tokenless schema/response compression for 60-90% LLM token savings",
+  "version": "0.8.0",
+  "description": "OpenClaw lifecycle adapter for Tokenless Protocol v2",
   "type": "module",
   "main": "dist/index.js",
   "openclaw": {
-    "extensions": ["./dist/index.js"]
+    "extensions": ["./dist/index.js"],
+    "compat": {
+      "pluginApi": ">=2026.4.22"
+    }
   },
   "scripts": {
     "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
     "build": "npm run clean && tsc --project tsconfig.json && cp ../common/hooks/tool_categories.json dist/"
   },
   "peerDependencies": {
-    "rtk": ">=0.35.0",
     "tokenless": ">=0.1.0"
   },
   "devDependencies": {

package/adapters/tokenless/qoder/.qoder-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "tokenless",
-  "version": "0.7.13",
+  "version": "0.8.0",
   "description": "Token-Less LLM Token Optimization Toolkit for Qoder CLI",
   "author": {
     "name": "Shile Zhang",

package/adapters/tokenless/qwencode/hooks/run-hook.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# run-hook.sh — Locate and exec a shared tokenless hook script.
+#
+# Prefer hooks from the wrapper's own adapter tree so concurrent RPM, Makefile,
+# and raw installations cannot cross-load another version. Hosts that copy the
+# wrapper into a detached cache still use the historical FHS fallbacks.
+#
+# Usage:    run-hook.sh <hook-script-basename> [args...]
+# Examples: run-hook.sh rewrite_hook.py
+#           run-hook.sh compress_response_hook.py
+#           run-hook.sh tool_ready_hook.sh
+#
+# Fail-open contract: any not-found / missing-interpreter condition emits
+# an empty JSON object on stdout and exits 0, so the host never blocks
+# on us.
+#
+# PreToolUse matcher overlap is by design: hooks.json registers both a
+# Bash-specific entry (rewrite) and a catch-all entry (tool-ready). Bash
+# tool calls therefore fire both — the host evaluates each matching
+# matcher independently, so this is the documented way to attach a
+# tool-specific hook alongside a global one. The timeout values are
+# upper bounds; observed runtimes are well under them.
+set -euo pipefail
+
+SCRIPT_DIR="$(CDPATH='' cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+SCRIPT="${1:?usage: run-hook.sh <hook-script-basename> [args...]}"
+shift
+
+fail_open() { echo "{}"; exit 0; }
+
+# Reject anything but a bare basename. Practical risk is low — hooks.json is
+# RPM-installed root:root 0644 and unwritable at runtime — but a defense-in-
+# depth check costs one line and stops any future caller from smuggling a
+# traversal segment through this argument.
+case "$SCRIPT" in
+    */*|"") fail_open ;;
+esac
+
+CANDIDATES=(
+    "${SCRIPT_DIR}/../../common/hooks/${SCRIPT}"
+    "/usr/local/share/anolisa/adapters/tokenless/common/hooks/${SCRIPT}"
+    "/usr/share/anolisa/adapters/tokenless/common/hooks/${SCRIPT}"
+    "${HOME}/.local/share/anolisa/adapters/tokenless/common/hooks/${SCRIPT}"
+)
+
+for candidate in "${CANDIDATES[@]}"; do
+    [ -f "$candidate" ] || continue
+    case "$candidate" in
+        *.py)
+            command -v python3 >/dev/null 2>&1 || fail_open
+            exec python3 "$candidate" "$@"
+            ;;
+        *.sh)
+            exec bash "$candidate" "$@"
+            ;;
+        *)
+            fail_open
+            ;;
+    esac
+done
+
+fail_open

package/adapters/tokenless/qwencode/qwen-extension.json

@@ -1,6 +1,6 @@
 {
   "name": "tokenless",
-  "version": "0.7.13",
+  "version": "0.8.0",
   "description": "Token-Less context compression for Qwen Code — RTK command rewriting, response/TOON/schema compression, and registered but hard-disabled Tool Ready",
   "author": { "name": "ANOLISA" },
   "license": "Apache-2.0",
@@ -29,8 +29,8 @@
             "type": "command",
             "name": "tokenless-rewrite",
             "description": "Rewrites shell commands via rtk for token savings",
-            "command": "bash \"${extensionPath}/hooks/run-hook.sh\" rewrite_hook.py",
-            "timeout": 5000,
+            "command": "bash \"${extensionPath}/hooks/run-hook.sh\" rewrite_hook.py --agent-id qwencode",
+            "timeout": 10000,
             "env": { "TOKENLESS_AGENT_ID": "qwencode" }
           }
         ]
@@ -44,7 +44,7 @@
             "type": "command",
             "name": "tokenless-compress-response",
             "description": "Compresses tool responses and encodes to TOON format",
-            "command": "bash \"${extensionPath}/hooks/run-hook.sh\" compress_response_hook.py",
+            "command": "bash \"${extensionPath}/hooks/run-hook.sh\" compress_response_hook.py --agent-id qwencode",
             "timeout": 10000,
             "env": { "TOKENLESS_AGENT_ID": "qwencode" }
           }

package/adapters/tokenless/qwenpaw/plugin.json

@@ -0,0 +1,17 @@
+{
+  "id": "tokenless",
+  "name": "Tokenless",
+  "version": "0.8.0",
+  "type": "general",
+  "description": "ANOLISA Tokenless: compress tool schemas and tool results before they reach the model",
+  "author": "ANOLISA",
+  "entry": {
+    "backend": "plugin.py"
+  },
+  "dependencies": [],
+  "qwenpaw_version": {
+    "min": "2.0.0",
+    "max": "3.0.0"
+  },
+  "meta": {}
+}

package/adapters/tokenless/qwenpaw/plugin.py

@@ -0,0 +1,390 @@
+"""Tokenless plugin for QwenPaw.
+
+Registers one AgentScope middleware that runs the complete Tokenless lifecycle
+in-process through ``anolisa_tokenless.TokenlessSdk``: tool schema compression
+before each model call, RTK command rewriting before ``execute_shell_command``,
+tool result compression after every tool call, and a marker-authorized
+``tokenless_retrieve`` tool.
+
+QwenPaw imports this file once at install time before the plugin's
+``requirements.txt`` is installed, so ``anolisa_tokenless`` is imported inside
+``register`` rather than at module level. Tools outside the built-in contract
+table are passed through untouched.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import AsyncGenerator, Callable
+from pathlib import Path
+from typing import Any
+
+from agentscope.message import TextBlock, ToolResultState
+from agentscope.middleware import MiddlewareBase
+from agentscope.tool import ToolChunk, ToolResponse
+
+logger = logging.getLogger(__name__)
+
+AGENT_ID = "qwenpaw"
+STATE_KEY = "anolisa_tokenless"
+RETRIEVE_TOOL = "tokenless_retrieve"
+RETRIEVE_DECLARATION = {
+    "type": "function",
+    "function": {
+        "name": RETRIEVE_TOOL,
+        "description": (
+            "Restore omitted content when needed. Pass only the 24-character hash "
+            "from a visible Tokenless recovery instruction, not the whole instruction."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "hash_or_marker": {
+                    "type": "string",
+                    "description": (
+                        "The 24-character hash from a visible recovery instruction; "
+                        "historical Tokenless markers are also accepted"
+                    ),
+                }
+            },
+            "required": ["hash_or_marker"],
+            "additionalProperties": False,
+        },
+    },
+}
+
+# tool name -> (content_origin wire value, RTK command field) for QwenPaw
+# 2.2.0's built-in tools. Core passes `file_content` through untouched, so a
+# tool absent from this table (skills, MCP servers, newer built-ins) falls
+# back to it and is never compressed or rewritten.
+_CONTRACTS: dict[str, tuple[str, str | None]] = {
+    "execute_shell_command": ("command_output", "command"),
+    **{
+        name: ("file_content", None)
+        for name in ("read_file", "recall_history", "view_image", "view_video")
+    },
+    **{
+        name: ("api_response", None)
+        for name in (
+            "glob_search",
+            "grep_search",
+            "ast_search",
+            "web_fetch",
+            "web_search",
+            "memory_search",
+            "get_current_time",
+            "get_token_usage",
+            "list_agents",
+            "check_agent_task",
+            "chat_with_agent",
+            "submit_to_agent",
+            "spawn_subagent",
+            "delegate_external_agent",
+            "browser",
+            "desktop_screenshot",
+            "run_tool_batch",
+            "send_file_to_user",
+            "set_user_timezone",
+            "activate_f1_exploration_mode",
+            "edit_file",
+            "write_file",
+            "append_file",
+            "materialize_skill",
+        )
+    },
+}
+_DEFAULT_CONTRACT: tuple[str, str | None] = ("file_content", None)
+
+# Every name the middleware and the retrieve tool import from
+# anolisa_tokenless. A released wheel that predates one of them imports fine
+# at registration and would fail at the first model call instead, so the
+# plugin refuses to register against such a wheel.
+_REQUIRED_SDK_NAMES = (
+    "Attribution",
+    "BeforeModelCapabilities",
+    "BeforeModelRequest",
+    "ContentOrigin",
+    "OutputOptimization",
+    "PostToolCapabilities",
+    "PostToolRequest",
+    "PreToolAction",
+    "PreToolCapabilities",
+    "PreToolRequest",
+    "RecoveryMethod",
+    "ResultKind",
+    "RetrieveRequest",
+    "TokenlessConfig",
+    "TokenlessError",
+    "TokenlessSdk",
+    "ToolResultStatus",
+)
+
+_SDKS: dict[str, Any] = {}
+
+
+def _sdk_for(data_dir: str) -> Any:
+    sdk = _SDKS.get(data_dir)
+    if sdk is None:
+        from anolisa_tokenless import TokenlessConfig, TokenlessSdk
+
+        sdk = TokenlessSdk(
+            TokenlessConfig(
+                data_dir=data_dir, rtk_enabled=True, retrieve_tool_name=RETRIEVE_TOOL
+            )
+        )
+        _SDKS[data_dir] = sdk
+    return sdk
+
+
+class TokenlessMiddleware(MiddlewareBase):
+    """Apply the Tokenless lifecycle to one QwenPaw agent request."""
+
+    def __init__(self, sdk: Any, data_dir: str) -> None:
+        self.sdk = sdk
+        self.data_dir = data_dir
+
+    async def on_model_call(
+        self,
+        agent: Any,
+        input_kwargs: dict[str, Any],
+        next_handler: Callable[..., Any],
+    ) -> Any:
+        from anolisa_tokenless import (
+            Attribution,
+            BeforeModelCapabilities,
+            BeforeModelRequest,
+            RecoveryMethod,
+        )
+
+        tools = [
+            tool
+            for tool in input_kwargs["tools"]
+            if tool.get("function", {}).get("name") != RETRIEVE_TOOL
+        ]
+        transformed = await self.sdk.before_model(
+            BeforeModelRequest(
+                tools=tuple(tools),
+                visible_context=json.dumps(
+                    input_kwargs["messages"], ensure_ascii=False, default=str
+                ),
+                capabilities=BeforeModelCapabilities(
+                    replace_tools=True, recovery=RecoveryMethod.tool(RETRIEVE_TOOL)
+                ),
+                attribution=Attribution(AGENT_ID, agent.state.session_id),
+            )
+        )
+        agent.state.middle_context[STATE_KEY] = {
+            "visible_markers": sorted(transformed.visible_markers),
+            "agent_id": AGENT_ID,
+            "data_dir": self.data_dir,
+        }
+        tools = list(transformed.tools)
+        tools.append(RETRIEVE_DECLARATION)
+        return await next_handler(**{**input_kwargs, "tools": tools})
+
+    async def on_acting(
+        self,
+        agent: Any,
+        input_kwargs: dict[str, Any],
+        next_handler: Callable[..., AsyncGenerator[Any, None]],
+    ) -> AsyncGenerator[Any, None]:
+        from anolisa_tokenless import (
+            Attribution,
+            OutputOptimization,
+            PreToolAction,
+            PreToolCapabilities,
+            PreToolRequest,
+        )
+
+        source = input_kwargs["tool_call"]
+        if source.name == RETRIEVE_TOOL:
+            async for item in next_handler(**input_kwargs):
+                yield item
+            return
+
+        origin, command_field = _CONTRACTS.get(source.name, _DEFAULT_CONTRACT)
+        attribution = Attribution(AGENT_ID, agent.state.session_id, source.id)
+        optimization = OutputOptimization.NONE
+        forwarded = source
+        if command_field is not None:
+            arguments = json.loads(source.input)
+            if not isinstance(arguments, dict):
+                raise TypeError(f"{source.name} arguments must be a JSON object")
+            transformed = await self.sdk.pre_tool(
+                PreToolRequest(
+                    tool_name=source.name,
+                    arguments=arguments,
+                    command_field=command_field,
+                    capabilities=PreToolCapabilities(
+                        replace_arguments=True, block_and_suggest=False
+                    ),
+                    attribution=attribution,
+                )
+            )
+            if transformed.action is PreToolAction.BLOCK_AND_SUGGEST:
+                raise RuntimeError(
+                    "Core returned block_and_suggest without host capability"
+                )
+            optimization = transformed.output_optimization
+            forwarded = source.model_copy(
+                update={
+                    "input": json.dumps(
+                        transformed.arguments,
+                        ensure_ascii=False,
+                        separators=(",", ":"),
+                    )
+                }
+            )
+        async for item in next_handler(**{**input_kwargs, "tool_call": forwarded}):
+            if isinstance(item, ToolResponse):
+                yield await self._after_response(
+                    item, source.name, origin, optimization, attribution
+                )
+            else:
+                yield item
+
+    async def _after_response(
+        self,
+        response: ToolResponse,
+        tool_name: str,
+        origin: str,
+        optimization: Any,
+        attribution: Any,
+    ) -> ToolResponse:
+        from anolisa_tokenless import (
+            ContentOrigin,
+            OutputOptimization,
+            PostToolCapabilities,
+            PostToolRequest,
+            RecoveryMethod,
+            ResultKind,
+            ToolResultStatus,
+        )
+
+        status = {
+            ToolResultState.SUCCESS: ToolResultStatus.SUCCESS,
+            ToolResultState.ERROR: ToolResultStatus.ERROR,
+            ToolResultState.INTERRUPTED: ToolResultStatus.INTERRUPTED,
+            ToolResultState.DENIED: ToolResultStatus.DENIED,
+        }[response.state]
+        # Recoverable compression needs the static retrieve tool; RTK output
+        # and failed results stay lossless-only, as in tokenless_agentscope.
+        recovery = (
+            RecoveryMethod.tool(RETRIEVE_TOOL)
+            if status is ToolResultStatus.SUCCESS
+            and optimization == OutputOptimization.NONE
+            else RecoveryMethod()
+        )
+        content = list(response.content)
+        extra_context: str | None = None
+        for index, block in enumerate(content):
+            if not isinstance(block, TextBlock):
+                continue
+            transformed = await self.sdk.post_tool(
+                PostToolRequest(
+                    result_kind=ResultKind.TOOL,
+                    tool_name=tool_name,
+                    content=block.text,
+                    status=status,
+                    content_origin=ContentOrigin(origin),
+                    output_optimization=optimization,
+                    capabilities=PostToolCapabilities(
+                        replace_output=True,
+                        recovery=recovery,
+                        replace_with_text=True,
+                    ),
+                    attribution=attribution,
+                )
+            )
+            extra_context = extra_context or transformed.additional_context
+            if transformed.output != block.text:
+                content[index] = block.model_copy(update={"text": transformed.output})
+        if extra_context is not None:
+            content.append(TextBlock(text=extra_context))
+        if content == response.content:
+            return response
+        return response.model_copy(update={"content": content})
+
+
+def _factory(ctx: Any, agent_config: Any) -> TokenlessMiddleware:
+    del agent_config
+    # TokenlessConfig rejects relative data directories.
+    data_dir = str(Path(ctx.workspace.workspace_dir).expanduser().resolve() / ".tokenless")
+    return TokenlessMiddleware(_sdk_for(data_dir), data_dir)
+
+
+async def tokenless_retrieve(hash_or_marker: str) -> ToolChunk:
+    """Restore omitted content behind a visible Tokenless recovery instruction.
+
+    Args:
+        hash_or_marker: The 24-character hash from a visible recovery
+            instruction; historical <<tokenless:HASH>> markers are accepted.
+    """
+    from anolisa_tokenless import Attribution, RetrieveRequest, TokenlessError
+    from qwenpaw.config.context import get_current_agent_state
+
+    state = get_current_agent_state()
+    saved = getattr(state, "middle_context", {}).get(STATE_KEY) if state else None
+    if not saved:
+        return ToolChunk(
+            content=[TextBlock(text="Tokenless has no marker state for this session")],
+            state=ToolResultState.ERROR,
+        )
+    try:
+        response = await _sdk_for(saved["data_dir"]).retrieve(
+            RetrieveRequest(
+                hash_or_marker,
+                frozenset(saved["visible_markers"]),
+                Attribution(saved["agent_id"], state.session_id),
+            )
+        )
+    except TokenlessError as error:
+        return ToolChunk(
+            content=[TextBlock(text=str(error))], state=ToolResultState.ERROR
+        )
+    return ToolChunk(
+        content=[TextBlock(text=response.payload)], state=ToolResultState.SUCCESS
+    )
+
+
+class TokenlessPlugin:
+    """QwenPaw plugin entry point."""
+
+    def register(self, api: Any) -> None:
+        import anolisa_tokenless
+
+        manifest = json.loads(
+            (Path(__file__).parent / "plugin.json").read_text(encoding="utf-8")
+        )
+        missing = [
+            name for name in _REQUIRED_SDK_NAMES if not hasattr(anolisa_tokenless, name)
+        ]
+        if missing:
+            logger.error(
+                "tokenless: anolisa_tokenless %s lacks %s required by plugin %s; "
+                "install the anolisa_tokenless wheel from the tokenless/v%s release "
+                "into QwenPaw's Python environment. Tokenless stays disabled.",
+                anolisa_tokenless.__version__,
+                ", ".join(missing),
+                manifest["version"],
+                manifest["version"],
+            )
+            return
+        if anolisa_tokenless.__version__ != manifest["version"]:
+            logger.warning(
+                "tokenless: plugin %s runs against anolisa_tokenless %s",
+                manifest["version"],
+                anolisa_tokenless.__version__,
+            )
+        api.register_middleware(_factory, priority=100)
+        api.register_tool(
+            tool_name=RETRIEVE_TOOL,
+            tool_func=tokenless_retrieve,
+            description="Restore content behind a Tokenless marker",
+            enabled=True,
+            tool_type="internal",
+        )
+
+
+plugin = TokenlessPlugin()

package/adapters/tokenless/qwenpaw/requirements.txt

@@ -0,0 +1,6 @@
+# Installed by QwenPaw's plugin loader into QwenPaw's own Python environment.
+# The native wheel is published only as a Tokenless GitHub Release asset; pip
+# picks the single line whose environment marker matches this host.
+anolisa-tokenless @ https://github.com/alibaba/anolisa/releases/download/tokenless/v0.8.0/anolisa_tokenless-0.8.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ; sys_platform == "linux" and platform_machine == "x86_64"
+anolisa-tokenless @ https://github.com/alibaba/anolisa/releases/download/tokenless/v0.8.0/anolisa_tokenless-0.8.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl ; sys_platform == "linux" and platform_machine == "aarch64"
+anolisa-tokenless @ https://github.com/alibaba/anolisa/releases/download/tokenless/v0.8.0/anolisa_tokenless-0.8.0-cp311-abi3-macosx_11_0_arm64.whl ; sys_platform == "darwin" and platform_machine == "arm64"