anolisa-tokenless 0.7.13 → 0.7.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -8
- package/adapters/tokenless/claude-code/.claude-plugin/plugin.json +1 -1
- package/adapters/tokenless/codex/.codex-plugin/plugin.json +5 -4
- package/adapters/tokenless/codex/README.md +22 -32
- package/adapters/tokenless/codex/hooks/hooks.json +3 -3
- package/adapters/tokenless/codex/scripts/response-diagnostics +170 -0
- package/adapters/tokenless/common/cosh-extension.json +1 -1
- package/adapters/tokenless/common/hooks/compress_response_hook.py +123 -264
- package/adapters/tokenless/common/hooks/compress_schema_hook.py +35 -31
- package/adapters/tokenless/common/hooks/compress_toon_hook.py +3 -0
- package/adapters/tokenless/common/hooks/hook_utils.py +70 -3
- package/adapters/tokenless/dsh/package.json +1 -1
- package/adapters/tokenless/hermes/__init__.py +2 -0
- package/adapters/tokenless/hermes/plugin.yaml +1 -1
- package/adapters/tokenless/manifest.json +2 -3
- package/adapters/tokenless/openclaw/openclaw.plugin.json +1 -1
- package/adapters/tokenless/openclaw/package.json +1 -1
- package/adapters/tokenless/qoder/.qoder-plugin/plugin.json +1 -1
- package/adapters/tokenless/qwencode/qwen-extension.json +1 -1
- package/package.json +5 -5
- package/adapters/tokenless/codex/scripts/compress-response +0 -445
|
@@ -231,9 +231,9 @@ def get_thresholds(tool_name: str) -> tuple[int, int, int]:
|
|
|
231
231
|
|
|
232
232
|
# -- Shared environment error patterns ----------------------------------------
|
|
233
233
|
#
|
|
234
|
-
# Superset of patterns from both the
|
|
234
|
+
# Superset of patterns from both the Codex diagnostics and shared hook adapters. Uses regex
|
|
235
235
|
# matching (case-insensitive) so patterns like "/bin/sh:.*: not found" work
|
|
236
|
-
# correctly. Both codex/scripts/
|
|
236
|
+
# correctly. Both codex/scripts/response-diagnostics and compress_response_hook
|
|
237
237
|
# import this list and the classify_env_error() function below.
|
|
238
238
|
|
|
239
239
|
ENV_PATTERNS: list[tuple[list[str], str, str]] = [
|
|
@@ -315,7 +315,7 @@ def classify_env_error(tool_response) -> tuple[str | None, str | None]:
|
|
|
315
315
|
Accepts either a parsed dict (with stderr/error/exit_code fields) or a
|
|
316
316
|
plain string. Returns (category_tag, fix_hint) or (None, None).
|
|
317
317
|
|
|
318
|
-
Shared by codex/scripts/
|
|
318
|
+
Shared by codex/scripts/response-diagnostics and compress_response_hook.
|
|
319
319
|
"""
|
|
320
320
|
if isinstance(tool_response, dict):
|
|
321
321
|
text = str(tool_response.get("stderr", "")) + str(tool_response.get("error", ""))
|
|
@@ -496,6 +496,73 @@ def run(args: list[str], input_data: str, timeout: int = 3) -> subprocess.Comple
|
|
|
496
496
|
return None
|
|
497
497
|
|
|
498
498
|
|
|
499
|
+
def build_compression_request(
|
|
500
|
+
content: str,
|
|
501
|
+
agent_id: str,
|
|
502
|
+
seam: str,
|
|
503
|
+
session_id: str = "",
|
|
504
|
+
tool_use_id: str = "",
|
|
505
|
+
tool_name: str = "",
|
|
506
|
+
replace_output: bool = False,
|
|
507
|
+
publish_retrieve_tool: bool = False,
|
|
508
|
+
replace_with_text: bool = False,
|
|
509
|
+
) -> dict:
|
|
510
|
+
"""Build a protocol-v1 CompressionRequest for ``tokenless compress``.
|
|
511
|
+
|
|
512
|
+
The adapter only copies the model-visible value and declares what its
|
|
513
|
+
host can do with the result (roadmap §4.5); all compression decisions
|
|
514
|
+
live behind the entry point.
|
|
515
|
+
"""
|
|
516
|
+
request = {
|
|
517
|
+
"protocol_version": 1,
|
|
518
|
+
"content": content,
|
|
519
|
+
"agent_id": agent_id,
|
|
520
|
+
"seam": seam,
|
|
521
|
+
"capabilities": {
|
|
522
|
+
"replace_output": replace_output,
|
|
523
|
+
"publish_retrieve_tool": publish_retrieve_tool,
|
|
524
|
+
"replace_with_text": replace_with_text,
|
|
525
|
+
},
|
|
526
|
+
}
|
|
527
|
+
if session_id:
|
|
528
|
+
request["session_id"] = session_id
|
|
529
|
+
if tool_use_id:
|
|
530
|
+
request["tool_use_id"] = tool_use_id
|
|
531
|
+
if tool_name:
|
|
532
|
+
request["tool_name"] = tool_name
|
|
533
|
+
return request
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def run_compress(
|
|
537
|
+
tokenless_bin: str, request: dict, timeout: int
|
|
538
|
+
) -> dict | None:
|
|
539
|
+
"""Run ``tokenless compress`` on one request; None on any failure.
|
|
540
|
+
|
|
541
|
+
The single Tokenless subprocess of a hook invocation (roadmap §5.6).
|
|
542
|
+
Fail-open: a dead binary, non-zero exit, or malformed stdout all
|
|
543
|
+
return None so the caller passes the original through.
|
|
544
|
+
"""
|
|
545
|
+
proc = run(
|
|
546
|
+
[tokenless_bin, "compress"],
|
|
547
|
+
json.dumps(request, ensure_ascii=False),
|
|
548
|
+
timeout=timeout,
|
|
549
|
+
)
|
|
550
|
+
if proc is None or proc.returncode != 0 or not proc.stdout.strip():
|
|
551
|
+
if proc is not None and proc.returncode != 0:
|
|
552
|
+
warn(f"tokenless compress exited {proc.returncode}")
|
|
553
|
+
return None
|
|
554
|
+
response = try_parse_json(proc.stdout.strip())
|
|
555
|
+
if not isinstance(response, dict):
|
|
556
|
+
warn("tokenless compress returned malformed output")
|
|
557
|
+
return None
|
|
558
|
+
if response.get("protocol_version") != 1:
|
|
559
|
+
# Version-skewed binary: never trust a response whose contract
|
|
560
|
+
# this adapter does not speak.
|
|
561
|
+
warn("tokenless compress returned an unsupported protocol version")
|
|
562
|
+
return None
|
|
563
|
+
return response
|
|
564
|
+
|
|
565
|
+
|
|
499
566
|
def detect_cosh_ng_runtime() -> tuple | None:
|
|
500
567
|
"""Detect if we are running under Cosh-NG and return its version.
|
|
501
568
|
|
|
@@ -184,6 +184,8 @@ _MIN_RESPONSE_LEN = 200
|
|
|
184
184
|
# saves only a few characters (observed ~0.3% below ~500 chars) while
|
|
185
185
|
# the per-event encode cost stays the same, so payloads under this
|
|
186
186
|
# threshold keep the response-compressed form and skip TOON entirely.
|
|
187
|
+
# Mirrors tokenless-runtime's MIN_TOON_CHARS, the default the
|
|
188
|
+
# compress-toon CLI applies; keep the two values in sync.
|
|
187
189
|
_MIN_TOON_CHARS = 500
|
|
188
190
|
|
|
189
191
|
_SKIP_TOOLS: set[str] = _SKIP_TOOLS_SHARED | {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"component": "tokenless",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.14",
|
|
4
4
|
"targets": {
|
|
5
5
|
"cosh": {
|
|
6
6
|
"compatibleVersions": "*",
|
|
@@ -99,8 +99,7 @@
|
|
|
99
99
|
"hooks": [
|
|
100
100
|
"tool-ready",
|
|
101
101
|
"rewrite",
|
|
102
|
-
"
|
|
103
|
-
"compress-toon"
|
|
102
|
+
"response-diagnostics"
|
|
104
103
|
]
|
|
105
104
|
},
|
|
106
105
|
"actions": {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "tokenless",
|
|
3
3
|
"name": "Tokenless",
|
|
4
|
-
"version": "0.7.
|
|
4
|
+
"version": "0.7.14",
|
|
5
5
|
"description": "Unified RTK command rewriting + response/TOON compression + registered but hard-disabled Tool Ready. Wraps tokenless and rtk system binaries via child_process — this is expected and not malicious.",
|
|
6
6
|
"activation": {
|
|
7
7
|
"onCapabilities": ["hook"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tokenless/openclaw-plugin",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.14",
|
|
4
4
|
"description": "Unified OpenClaw plugin — RTK command rewriting + tokenless schema/response compression for 60-90% LLM token savings",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tokenless",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.14",
|
|
4
4
|
"description": "Token-Less context compression for Qwen Code — RTK command rewriting, response/TOON/schema compression, and registered but hard-disabled Tool Ready",
|
|
5
5
|
"author": { "name": "ANOLISA" },
|
|
6
6
|
"license": "Apache-2.0",
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "anolisa-tokenless",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.7.
|
|
4
|
+
"version": "0.7.14",
|
|
5
5
|
"description": "Token-Less — LLM token optimization toolkit (schema/response compression, command rewriting, tool readiness)",
|
|
6
6
|
"license": "Apache-2.0",
|
|
7
7
|
"repository": {
|
|
@@ -44,10 +44,10 @@
|
|
|
44
44
|
"arm64"
|
|
45
45
|
],
|
|
46
46
|
"optionalDependencies": {
|
|
47
|
-
"@anolisa/tokenless-linux-x64": "0.7.
|
|
48
|
-
"@anolisa/tokenless-linux-arm64": "0.7.
|
|
49
|
-
"@anolisa/tokenless-darwin-x64": "0.7.
|
|
50
|
-
"@anolisa/tokenless-darwin-arm64": "0.7.
|
|
47
|
+
"@anolisa/tokenless-linux-x64": "0.7.14",
|
|
48
|
+
"@anolisa/tokenless-linux-arm64": "0.7.14",
|
|
49
|
+
"@anolisa/tokenless-darwin-x64": "0.7.14",
|
|
50
|
+
"@anolisa/tokenless-darwin-arm64": "0.7.14"
|
|
51
51
|
},
|
|
52
52
|
"publishConfig": {
|
|
53
53
|
"registry": "https://registry.npmjs.org/",
|
|
@@ -1,445 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Codex PostToolUse compression hook for the Tokenless plugin.
|
|
3
|
-
|
|
4
|
-
Reads a PostToolUse hook request from stdin, compresses the tool response
|
|
5
|
-
via ``tokenless compress-response`` and optional TOON encoding, then
|
|
6
|
-
injects a compressed summary and environment error classification as
|
|
7
|
-
``additionalContext``.
|
|
8
|
-
|
|
9
|
-
Protocol:
|
|
10
|
-
stdin ← codex PostToolUse JSON (tool_name, tool_response, ...)
|
|
11
|
-
stdout → {"hookSpecificOutput": {"hookEventName": "PostToolUse",
|
|
12
|
-
"additionalContext": "..."}}
|
|
13
|
-
|
|
14
|
-
Design notes for Codex:
|
|
15
|
-
- PostToolUse CANNOT suppress the original tool output (``suppressOutput``
|
|
16
|
-
is rejected by the codex hook engine for this event).
|
|
17
|
-
- ``additionalContext`` is additive — the model sees both the original
|
|
18
|
-
output AND the injected context.
|
|
19
|
-
- Therefore we inject a compressed *summary* rather than duplicating the
|
|
20
|
-
full content, plus environment error classification with fix hints.
|
|
21
|
-
- For very large JSON responses (> 4 KB), we include the full compressed
|
|
22
|
-
content so the model has a compact alternative to the raw output.
|
|
23
|
-
|
|
24
|
-
Fail-open: every error path silently passes through (exit 0 with empty
|
|
25
|
-
stdout) so the agent session is never blocked by a compression failure.
|
|
26
|
-
"""
|
|
27
|
-
|
|
28
|
-
import json
|
|
29
|
-
import os
|
|
30
|
-
import shutil
|
|
31
|
-
import subprocess
|
|
32
|
-
import sys
|
|
33
|
-
from typing import Optional
|
|
34
|
-
|
|
35
|
-
# Resolve shared hook utilities (common/hooks/) with FHS fallback paths.
|
|
36
|
-
# Primary: relative path (works for source-tree and FHS/RPM install).
|
|
37
|
-
# Fallbacks: system and user FHS paths (works when Codex caches only the
|
|
38
|
-
# plugin subdirectory and the relative path resolves outside the cache).
|
|
39
|
-
#
|
|
40
|
-
# Trust model (aligned with bash is_trusted_file / Rust is_trusted_path):
|
|
41
|
-
# - System FHS paths (/usr/share, /usr/local/share) are unconditional.
|
|
42
|
-
# - User-relative paths use passwd DB home (NOT $HOME — env-controllable),
|
|
43
|
-
# with ownership and parent-dir permission checks.
|
|
44
|
-
# - Relative ../../common/hooks is trusted only if the resolved target
|
|
45
|
-
# exists and is under a system prefix or a passwd-validated home path.
|
|
46
|
-
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def _is_trusted_dir(path: str) -> bool:
|
|
50
|
-
"""Check whether a directory is trusted for Python module imports.
|
|
51
|
-
|
|
52
|
-
Mirrors the trust criteria in bash tool_ready_hook.sh (is_trusted_file)
|
|
53
|
-
and Rust env_check.rs (is_trusted_path): system paths unconditional,
|
|
54
|
-
user paths validated via passwd + ownership + parent permission checks.
|
|
55
|
-
|
|
56
|
-
NOTE: Callers should pass realpath-resolved paths (not just abspath)
|
|
57
|
-
so that symlink targets are validated before the trust check — a
|
|
58
|
-
symlinked /usr/share/anolisa → /tmp/ would bypass the prefix check
|
|
59
|
-
if only abspath is used.
|
|
60
|
-
"""
|
|
61
|
-
# System FHS prefixes are always trusted
|
|
62
|
-
for prefix in ("/usr/share/", "/usr/local/share/", "/usr/libexec/", "/usr/lib/anolisa/"):
|
|
63
|
-
if path.startswith(prefix):
|
|
64
|
-
return True
|
|
65
|
-
# For paths outside system prefixes (e.g. source-tree cloned to
|
|
66
|
-
# /opt/anolisa/), trust them if owned by current uid or root AND
|
|
67
|
-
# parent directory is not world-writable. This covers developer
|
|
68
|
-
# worktrees without requiring the path to be under passwd home.
|
|
69
|
-
try:
|
|
70
|
-
st = os.stat(path)
|
|
71
|
-
except OSError:
|
|
72
|
-
return False
|
|
73
|
-
if st.st_uid != os.getuid() and st.st_uid != 0:
|
|
74
|
-
# Not owned by us or root — refuse. Home paths must be owned by
|
|
75
|
-
# us or root; source-tree paths outside system prefixes are
|
|
76
|
-
# already covered by the ownership check above (current uid/root).
|
|
77
|
-
return False
|
|
78
|
-
# Ownership OK — check parent directory is not world-writable
|
|
79
|
-
parent = os.path.dirname(path)
|
|
80
|
-
try:
|
|
81
|
-
pst = os.stat(parent)
|
|
82
|
-
except OSError:
|
|
83
|
-
return False
|
|
84
|
-
if pst.st_uid != os.getuid() and pst.st_uid != 0:
|
|
85
|
-
return False
|
|
86
|
-
if pst.st_mode & 0o002: # world-writable
|
|
87
|
-
return False
|
|
88
|
-
return True
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
# Resolve real home from passwd DB for user-install fallback path.
|
|
92
|
-
# Falls back to "" (empty) when unavailable — candidate is then skipped.
|
|
93
|
-
try:
|
|
94
|
-
import pwd as _pwd
|
|
95
|
-
_REAL_HOME = _pwd.getpwuid(os.getuid()).pw_dir
|
|
96
|
-
except (ImportError, KeyError):
|
|
97
|
-
_REAL_HOME = ""
|
|
98
|
-
if not os.path.isabs(_REAL_HOME):
|
|
99
|
-
_REAL_HOME = ""
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
def _user_path(*parts: str) -> str:
|
|
103
|
-
return os.path.join(_REAL_HOME, *parts) if _REAL_HOME else ""
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
_HOOK_UTILS_CANDIDATES = [
|
|
107
|
-
os.path.join(_HERE, "..", "..", "common", "hooks"), # source-tree / FHS
|
|
108
|
-
"/usr/share/anolisa/adapters/tokenless/common/hooks", # RPM system
|
|
109
|
-
"/usr/local/share/anolisa/adapters/tokenless/common/hooks", # manual system
|
|
110
|
-
os.path.join(_REAL_HOME, ".local", "share",
|
|
111
|
-
"anolisa", "adapters", "tokenless", "common", "hooks") if _REAL_HOME else "",
|
|
112
|
-
]
|
|
113
|
-
_HOOK_UTILS_RESOLVED = ""
|
|
114
|
-
for _C in _HOOK_UTILS_CANDIDATES:
|
|
115
|
-
if _C and _is_trusted_dir(os.path.realpath(_C)) and os.path.isdir(_C):
|
|
116
|
-
_HOOK_UTILS_RESOLVED = os.path.realpath(_C)
|
|
117
|
-
sys.path.insert(0, _C)
|
|
118
|
-
break
|
|
119
|
-
|
|
120
|
-
if _HOOK_UTILS_RESOLVED:
|
|
121
|
-
from hook_utils import ( # noqa: E402
|
|
122
|
-
skip_silent as _skip,
|
|
123
|
-
try_parse_json as _try_parse_json,
|
|
124
|
-
warn as _warn,
|
|
125
|
-
classify_env_error as _classify_env_error,
|
|
126
|
-
get_thresholds as _get_thresholds,
|
|
127
|
-
SKIP_TOOLS as _SKIP_TOOLS_SHARED,
|
|
128
|
-
)
|
|
129
|
-
else:
|
|
130
|
-
# Fail-open: no trusted hook_utils found → provide inline fallbacks
|
|
131
|
-
# so the hook script exits 0 (passthrough) rather than ImportError → exit 1.
|
|
132
|
-
def _skip() -> None: # type: ignore[assignment,misc]
|
|
133
|
-
sys.exit(0)
|
|
134
|
-
|
|
135
|
-
def _try_parse_json(data: str) -> object | None: # type: ignore[assignment,misc]
|
|
136
|
-
try:
|
|
137
|
-
return json.loads(data)
|
|
138
|
-
except (json.JSONDecodeError, ValueError):
|
|
139
|
-
return None
|
|
140
|
-
|
|
141
|
-
def _warn(msg: str) -> None: # type: ignore[assignment,misc]
|
|
142
|
-
print(f"[tokenless] WARNING: {msg}", file=sys.stderr)
|
|
143
|
-
|
|
144
|
-
def _classify_env_error(tool_response) -> tuple[str | None, str | None]: # type: ignore[assignment,misc]
|
|
145
|
-
# Classification unavailable without hook_utils — return no-op.
|
|
146
|
-
# Caller handles (None, None) by skipping env error injection.
|
|
147
|
-
return None, None
|
|
148
|
-
|
|
149
|
-
def _get_thresholds(tool_name: str) -> tuple[int, int, int]: # type: ignore[assignment,misc]
|
|
150
|
-
# Hardcoded defaults mirroring hook_utils.get_thresholds().
|
|
151
|
-
# Keep in sync with common/hooks/hook_utils.py thresholds.
|
|
152
|
-
return (65_536, 128, 8) if tool_name in {"Bash", "bash", "Shell", "shell", "exec", "terminal"} else (1_048_576, 65_536, 32)
|
|
153
|
-
|
|
154
|
-
_SKIP_TOOLS_SHARED: set[str] = { # type: ignore[assignment,misc]
|
|
155
|
-
"Read", "read", "read_file", "read_many_files",
|
|
156
|
-
"Glob", "glob", "list_directory",
|
|
157
|
-
"Grep", "grep", "grep_search", "search_files",
|
|
158
|
-
"Lsp", "lsp",
|
|
159
|
-
"NotebookRead", "notebook_read", "notebookread",
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
# ---------------------------------------------------------------------------
|
|
163
|
-
# constants
|
|
164
|
-
# ---------------------------------------------------------------------------
|
|
165
|
-
|
|
166
|
-
AGENT_ID = os.environ.get("TOKENLESS_AGENT_ID", "codex")
|
|
167
|
-
MIN_RESPONSE_CHARS: int = 500
|
|
168
|
-
LARGE_RESPONSE_CHARS: int = 4000
|
|
169
|
-
# TOON on small JSON saves only a few characters (observed ~0.3% below
|
|
170
|
-
# ~500 chars) while the per-event encode cost stays the same, so payloads
|
|
171
|
-
# under this threshold keep the compressed form and skip the TOON pass.
|
|
172
|
-
MIN_TOON_CHARS: int = 500
|
|
173
|
-
|
|
174
|
-
# 3-layer compression strategy:
|
|
175
|
-
# Layer 1: Content retrieval + task management → skip all compression
|
|
176
|
-
# Layer 2: Shell/exec → moderate truncation (64K/128/8)
|
|
177
|
-
# Layer 3: API/structured → zero-truncation (1M/64K/32)
|
|
178
|
-
|
|
179
|
-
# Layer 1: content retrieval (shared) + codex-specific task management
|
|
180
|
-
SKIP_TOOLS: set[str] = _SKIP_TOOLS_SHARED | {
|
|
181
|
-
"TodoWrite", "Task", "TaskStatus",
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
# Cap for compressed content included in additionalContext (large responses only).
|
|
185
|
-
# Layer 2 thresholds (64K strings) preserve 95% of real shell output; Layer 3
|
|
186
|
-
# zero-truncation (1M strings) can still produce large payloads for API tools.
|
|
187
|
-
COMPRESSED_CONTENT_CAP: int = 32_768
|
|
188
|
-
|
|
189
|
-
# -- tokenless binary search paths (ANOLISA FHS spec) ------------------------
|
|
190
|
-
# Codex can cache this script without common/hooks. Keep this standalone list
|
|
191
|
-
# aligned with hook_utils.py::_known_binary_paths.
|
|
192
|
-
|
|
193
|
-
_TOKENLESS_FALLBACK = "/usr/bin/tokenless"
|
|
194
|
-
_TOKENLESS_SYSTEM_FALLBACK = "/usr/local/bin/tokenless"
|
|
195
|
-
_TOKENLESS_LOCAL_BIN = _user_path(".local", "bin", "tokenless")
|
|
196
|
-
_TOKENLESS_LOCAL_SHARE = _user_path(
|
|
197
|
-
".local", "share", "anolisa", "tokenless", "tokenless"
|
|
198
|
-
)
|
|
199
|
-
_TOKENLESS_LOCAL_LIB = _user_path(
|
|
200
|
-
".local", "lib", "anolisa", "tokenless", "tokenless"
|
|
201
|
-
)
|
|
202
|
-
|
|
203
|
-
# ---------------------------------------------------------------------------
|
|
204
|
-
# helpers
|
|
205
|
-
# ---------------------------------------------------------------------------
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
def _find_tokenless() -> Optional[str]:
|
|
209
|
-
"""Locate the tokenless CLI binary."""
|
|
210
|
-
explicit = os.environ.get("TOKENLESS_BIN", "")
|
|
211
|
-
if explicit and os.path.isfile(explicit) and os.access(explicit, os.X_OK):
|
|
212
|
-
return explicit
|
|
213
|
-
|
|
214
|
-
path = shutil.which("tokenless")
|
|
215
|
-
if path:
|
|
216
|
-
return path
|
|
217
|
-
|
|
218
|
-
for fp in (
|
|
219
|
-
_TOKENLESS_LOCAL_BIN,
|
|
220
|
-
_TOKENLESS_SYSTEM_FALLBACK,
|
|
221
|
-
_TOKENLESS_FALLBACK,
|
|
222
|
-
_TOKENLESS_LOCAL_SHARE,
|
|
223
|
-
_TOKENLESS_LOCAL_LIB,
|
|
224
|
-
):
|
|
225
|
-
if os.path.isfile(fp) and os.access(fp, os.X_OK):
|
|
226
|
-
return fp
|
|
227
|
-
return None
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
def _is_json_serializable(obj) -> bool:
|
|
231
|
-
"""Check if a value is a JSON object or array (compressible)."""
|
|
232
|
-
return isinstance(obj, (dict, list))
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
def _extract_text_for_compression(tool_response) -> Optional[str]:
|
|
236
|
-
"""Extract a JSON string from tool_response for compression."""
|
|
237
|
-
if isinstance(tool_response, str):
|
|
238
|
-
return tool_response
|
|
239
|
-
if isinstance(tool_response, (dict, list)):
|
|
240
|
-
return json.dumps(tool_response, separators=(",", ":"), ensure_ascii=False)
|
|
241
|
-
return None
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
def _run_tokenless(tokenless_bin: str, subcommand: str, input_data: str,
|
|
245
|
-
session_id: str = "", tool_use_id: str = "",
|
|
246
|
-
timeout: int = 10,
|
|
247
|
-
extra_args: Optional[list[str]] = None) -> Optional[str]:
|
|
248
|
-
"""Run a tokenless subcommand with input, return stdout or None on failure."""
|
|
249
|
-
cmd = [tokenless_bin, subcommand, "--agent-id", AGENT_ID]
|
|
250
|
-
if extra_args:
|
|
251
|
-
cmd.extend(extra_args)
|
|
252
|
-
if session_id:
|
|
253
|
-
cmd.extend(["--session-id", session_id])
|
|
254
|
-
if tool_use_id:
|
|
255
|
-
cmd.extend(["--tool-use-id", tool_use_id])
|
|
256
|
-
|
|
257
|
-
try:
|
|
258
|
-
proc = subprocess.run(
|
|
259
|
-
cmd,
|
|
260
|
-
input=input_data,
|
|
261
|
-
capture_output=True,
|
|
262
|
-
text=True,
|
|
263
|
-
timeout=timeout,
|
|
264
|
-
)
|
|
265
|
-
if proc.returncode == 0 and proc.stdout.strip():
|
|
266
|
-
return proc.stdout.strip()
|
|
267
|
-
if proc.returncode != 0 and proc.stderr:
|
|
268
|
-
_warn(f"{subcommand}: {proc.stderr.strip()[:200]}")
|
|
269
|
-
except subprocess.TimeoutExpired:
|
|
270
|
-
_warn(f"{subcommand} timed out after {timeout}s")
|
|
271
|
-
except OSError as e:
|
|
272
|
-
_warn(f"{subcommand} OS error: {e}")
|
|
273
|
-
return None
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
# ---------------------------------------------------------------------------
|
|
277
|
-
# main
|
|
278
|
-
# ---------------------------------------------------------------------------
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
def main() -> None:
|
|
282
|
-
# 1. Locate tokenless binary (fail-open: skip if not installed)
|
|
283
|
-
tokenless_bin = _find_tokenless()
|
|
284
|
-
if not tokenless_bin:
|
|
285
|
-
_skip()
|
|
286
|
-
|
|
287
|
-
# 2. Read and parse stdin JSON from codex
|
|
288
|
-
try:
|
|
289
|
-
raw = sys.stdin.read()
|
|
290
|
-
if not raw.strip():
|
|
291
|
-
_skip()
|
|
292
|
-
input_data = json.loads(raw)
|
|
293
|
-
except (json.JSONDecodeError, EOFError, ValueError):
|
|
294
|
-
_skip()
|
|
295
|
-
|
|
296
|
-
# 3. Extract fields
|
|
297
|
-
tool_name = input_data.get("tool_name", "")
|
|
298
|
-
tool_response = input_data.get("tool_response")
|
|
299
|
-
session_id = input_data.get("session_id", "")
|
|
300
|
-
tool_use_id = input_data.get("tool_use_id", "")
|
|
301
|
-
|
|
302
|
-
# 4. Layer 1: Content retrieval + task management → skip entirely
|
|
303
|
-
if tool_name in SKIP_TOOLS:
|
|
304
|
-
_skip()
|
|
305
|
-
|
|
306
|
-
# 5. Skip empty or trivial responses
|
|
307
|
-
if tool_response is None:
|
|
308
|
-
_skip()
|
|
309
|
-
|
|
310
|
-
raw_text = _extract_text_for_compression(tool_response)
|
|
311
|
-
if not raw_text or raw_text in ("{}", "[]", '""'):
|
|
312
|
-
_skip()
|
|
313
|
-
|
|
314
|
-
# 6. Detect environment errors (always, regardless of size)
|
|
315
|
-
env_category, env_hint = _classify_env_error(tool_response)
|
|
316
|
-
|
|
317
|
-
# 7. Skip compression for small responses, but still report env errors
|
|
318
|
-
if len(raw_text) < MIN_RESPONSE_CHARS:
|
|
319
|
-
if env_category:
|
|
320
|
-
response = {
|
|
321
|
-
"hookSpecificOutput": {
|
|
322
|
-
"hookEventName": "PostToolUse",
|
|
323
|
-
"additionalContext": (
|
|
324
|
-
f"[tokenless:env:{env_category}] {env_hint} "
|
|
325
|
-
f"Do NOT retry the same command — fix the environment first."
|
|
326
|
-
),
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
print(json.dumps(response, ensure_ascii=False))
|
|
330
|
-
_skip()
|
|
331
|
-
|
|
332
|
-
# 8. Determine JSON status (needed for compression pipeline)
|
|
333
|
-
parsed = _try_parse_json(raw_text)
|
|
334
|
-
is_json = parsed is not None and _is_json_serializable(parsed)
|
|
335
|
-
|
|
336
|
-
# 9. Compression pipeline: 3-layer dispatch thresholds
|
|
337
|
-
# Layer 1 (content retrieval): already skipped above
|
|
338
|
-
# Layer 2 (shell/exec): moderate truncation (64K/128/8)
|
|
339
|
-
# Layer 3 (API/structured): zero-truncation (1M/64K/32)
|
|
340
|
-
compressed_text: Optional[str] = None
|
|
341
|
-
toon_text: Optional[str] = None
|
|
342
|
-
|
|
343
|
-
# -- Build compression input --
|
|
344
|
-
# If the response is plain text (not JSON), wrap it as a structured
|
|
345
|
-
# dict so the compression pipeline can still run and record stats.
|
|
346
|
-
# Otherwise bare strings like git log output would be silently skipped
|
|
347
|
-
# and never appear in tokenless stats, unlike openclaw/hermes where the
|
|
348
|
-
# agent framework natively wraps Bash output in {"stdout":...,...}.
|
|
349
|
-
if is_json:
|
|
350
|
-
compression_input = raw_text
|
|
351
|
-
else:
|
|
352
|
-
compression_input = json.dumps(
|
|
353
|
-
{"stdout": raw_text}, separators=(",", ":"), ensure_ascii=False
|
|
354
|
-
)
|
|
355
|
-
|
|
356
|
-
# -- Determine thresholds based on tool category --
|
|
357
|
-
thresholds = _get_thresholds(tool_name)
|
|
358
|
-
|
|
359
|
-
# -- Response compression --
|
|
360
|
-
compressed_text = _run_tokenless(
|
|
361
|
-
tokenless_bin, "compress-response",
|
|
362
|
-
compression_input, session_id, tool_use_id, timeout=10,
|
|
363
|
-
extra_args=[
|
|
364
|
-
"--truncate-strings-at", str(thresholds[0]),
|
|
365
|
-
"--truncate-arrays-at", str(thresholds[1]),
|
|
366
|
-
"--max-depth", str(thresholds[2]),
|
|
367
|
-
],
|
|
368
|
-
)
|
|
369
|
-
if compressed_text and len(compressed_text) >= len(compression_input):
|
|
370
|
-
compressed_text = None # no savings → skip
|
|
371
|
-
|
|
372
|
-
# -- TOON encoding (if still valid JSON after compression) --
|
|
373
|
-
if compressed_text:
|
|
374
|
-
toon_input = compressed_text
|
|
375
|
-
else:
|
|
376
|
-
toon_input = compression_input
|
|
377
|
-
|
|
378
|
-
toon_parsed = _try_parse_json(toon_input)
|
|
379
|
-
if toon_parsed is not None and len(toon_input) >= MIN_TOON_CHARS:
|
|
380
|
-
toon_text = _run_tokenless(
|
|
381
|
-
tokenless_bin, "compress-toon",
|
|
382
|
-
toon_input, session_id, tool_use_id, timeout=10,
|
|
383
|
-
)
|
|
384
|
-
if toon_text and len(toon_text) >= len(toon_input):
|
|
385
|
-
toon_text = None # no savings → skip
|
|
386
|
-
|
|
387
|
-
# 10. Build additionalContext
|
|
388
|
-
parts: list[str] = []
|
|
389
|
-
|
|
390
|
-
original_chars = len(raw_text)
|
|
391
|
-
final_text = toon_text or compressed_text
|
|
392
|
-
is_large = len(raw_text) >= LARGE_RESPONSE_CHARS
|
|
393
|
-
|
|
394
|
-
if final_text:
|
|
395
|
-
final_chars = len(final_text)
|
|
396
|
-
savings_pct = round((1 - final_chars / max(original_chars, 1)) * 100)
|
|
397
|
-
parts.append(
|
|
398
|
-
f"[tokenless:compressed] {tool_name}: "
|
|
399
|
-
f"{original_chars:,} → {final_chars:,} chars ({savings_pct}% reduction)"
|
|
400
|
-
)
|
|
401
|
-
else:
|
|
402
|
-
# No compression savings
|
|
403
|
-
if is_large:
|
|
404
|
-
parts.append(
|
|
405
|
-
f"[tokenless] {tool_name}: {original_chars:,} chars, "
|
|
406
|
-
f"no compression savings achieved"
|
|
407
|
-
)
|
|
408
|
-
|
|
409
|
-
# Environment error hints
|
|
410
|
-
if env_category:
|
|
411
|
-
parts.append(
|
|
412
|
-
f"[tokenless:env:{env_category}] {env_hint} "
|
|
413
|
-
f"Do NOT retry the same command — fix the environment first."
|
|
414
|
-
)
|
|
415
|
-
|
|
416
|
-
# For large responses, include the compressed content so the model has a
|
|
417
|
-
# compact alternative to the raw output.
|
|
418
|
-
if final_text and is_large:
|
|
419
|
-
parts.append("--- compressed content ---")
|
|
420
|
-
if len(final_text) > COMPRESSED_CONTENT_CAP:
|
|
421
|
-
parts.append(final_text[:COMPRESSED_CONTENT_CAP])
|
|
422
|
-
parts.append(
|
|
423
|
-
f"... (compressed content truncated at {COMPRESSED_CONTENT_CAP:,} chars)"
|
|
424
|
-
)
|
|
425
|
-
else:
|
|
426
|
-
parts.append(final_text)
|
|
427
|
-
parts.append("--- end compressed content ---")
|
|
428
|
-
|
|
429
|
-
if not parts:
|
|
430
|
-
_skip()
|
|
431
|
-
|
|
432
|
-
additional_context = "\n".join(parts)
|
|
433
|
-
|
|
434
|
-
# 11. Output codex hook response
|
|
435
|
-
response = {
|
|
436
|
-
"hookSpecificOutput": {
|
|
437
|
-
"hookEventName": "PostToolUse",
|
|
438
|
-
"additionalContext": additional_context,
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
print(json.dumps(response, ensure_ascii=False))
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
if __name__ == "__main__":
|
|
445
|
-
main()
|