anolisa-tokenless 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/README.md +770 -0
- package/adapters/tokenless/claude-code/.claude-plugin/marketplace.json +15 -0
- package/adapters/tokenless/claude-code/.claude-plugin/plugin.json +9 -0
- package/adapters/tokenless/claude-code/hooks/hooks.json +38 -0
- package/adapters/tokenless/claude-code/scripts/detect.sh +193 -0
- package/adapters/tokenless/claude-code/scripts/install.sh +92 -0
- package/adapters/tokenless/claude-code/scripts/uninstall.sh +50 -0
- package/adapters/tokenless/codex/.codex-plugin/plugin.json +20 -0
- package/adapters/tokenless/codex/README.md +160 -0
- package/adapters/tokenless/codex/hooks/hooks.json +52 -0
- package/adapters/tokenless/codex/scripts/_common.sh +25 -0
- package/adapters/tokenless/codex/scripts/check-tokenless +94 -0
- package/adapters/tokenless/codex/scripts/compress-response +441 -0
- package/adapters/tokenless/codex/scripts/detect.sh +61 -0
- package/adapters/tokenless/codex/scripts/install.sh +151 -0
- package/adapters/tokenless/codex/scripts/rewrite-hook +282 -0
- package/adapters/tokenless/codex/scripts/tool-ready +303 -0
- package/adapters/tokenless/codex/scripts/uninstall.sh +78 -0
- package/adapters/tokenless/common/commands/tokenless-stats.toml +2 -0
- package/adapters/tokenless/common/cosh-extension.json +65 -0
- package/adapters/tokenless/common/hooks/compress_response_hook.py +487 -0
- package/adapters/tokenless/common/hooks/compress_schema_hook.py +144 -0
- package/adapters/tokenless/common/hooks/compress_toon_hook.py +160 -0
- package/adapters/tokenless/common/hooks/hook_utils.py +584 -0
- package/adapters/tokenless/common/hooks/rewrite_hook.py +223 -0
- package/adapters/tokenless/common/hooks/run-hook.sh +62 -0
- package/adapters/tokenless/common/hooks/tool_categories.json +99 -0
- package/adapters/tokenless/common/hooks/tool_ready_hook.sh +571 -0
- package/adapters/tokenless/common/tokenless-env-fix.sh +730 -0
- package/adapters/tokenless/common/tool-ready-spec.json +113 -0
- package/adapters/tokenless/dsh/cordis.patch.yml +8 -0
- package/adapters/tokenless/dsh/dist/index.js +399 -0
- package/adapters/tokenless/dsh/package.json +26 -0
- package/adapters/tokenless/hermes/__init__.py +572 -0
- package/adapters/tokenless/hermes/plugin.yaml +9 -0
- package/adapters/tokenless/hermes/scripts/detect.sh +80 -0
- package/adapters/tokenless/hermes/scripts/install.sh +60 -0
- package/adapters/tokenless/hermes/scripts/uninstall.sh +45 -0
- package/adapters/tokenless/manifest.json +147 -0
- package/adapters/tokenless/openclaw/dist/index.d.ts +27 -0
- package/adapters/tokenless/openclaw/dist/index.js +598 -0
- package/adapters/tokenless/openclaw/dist/tool_categories.json +99 -0
- package/adapters/tokenless/openclaw/index.ts +720 -0
- package/adapters/tokenless/openclaw/openclaw.plugin.json +40 -0
- package/adapters/tokenless/openclaw/package.json +24 -0
- package/adapters/tokenless/openclaw/scripts/detect.sh +102 -0
- package/adapters/tokenless/openclaw/scripts/install.sh +75 -0
- package/adapters/tokenless/openclaw/scripts/uninstall.sh +46 -0
- package/adapters/tokenless/openclaw/tsconfig.json +13 -0
- package/adapters/tokenless/opencode/plugin.js +246 -0
- package/adapters/tokenless/opencode/scripts/detect.sh +38 -0
- package/adapters/tokenless/opencode/scripts/install.sh +56 -0
- package/adapters/tokenless/opencode/scripts/uninstall.sh +29 -0
- package/adapters/tokenless/qoder/.qoder-plugin/plugin.json +11 -0
- package/adapters/tokenless/qoder/commands/tokenless-stats.md +8 -0
- package/adapters/tokenless/qoder/hooks/hooks.json +36 -0
- package/adapters/tokenless/qoder/hooks/run-hook.sh +51 -0
- package/adapters/tokenless/qoder/scripts/detect.sh +35 -0
- package/adapters/tokenless/qoder/scripts/install.sh +136 -0
- package/adapters/tokenless/qoder/scripts/uninstall.sh +106 -0
- package/adapters/tokenless/qwencode/qwen-extension.json +69 -0
- package/adapters/tokenless/qwencode/scripts/detect.sh +125 -0
- package/adapters/tokenless/qwencode/scripts/install.sh +128 -0
- package/adapters/tokenless/qwencode/scripts/uninstall.sh +62 -0
- package/bin/rtk +6 -0
- package/bin/tokenless +6 -0
- package/bin/toon +6 -0
- package/package.json +57 -0
- package/scripts/postinstall.js +208 -0
|
@@ -0,0 +1,572 @@
|
|
|
1
|
+
"""Token-Less Plugin for Hermes Agent.
|
|
2
|
+
|
|
3
|
+
Combines multiple context-compression strategies into a single plugin:
|
|
4
|
+
|
|
5
|
+
1. **Response compression** — ``transform_tool_result`` : compresses tool
|
|
6
|
+
results via ``tokenless compress-response`` with per-layer thresholds
|
|
7
|
+
(moderate for shell, zero-truncation for API tools), stripping debug
|
|
8
|
+
fields, nulls, empty values.
|
|
9
|
+
2. **TOON encoding** — ``transform_tool_result`` : pipeline step after
|
|
10
|
+
response compression; re-encodes JSON results to TOON format via
|
|
11
|
+
``tokenless compress-toon`` for additional token savings (15-40%)
|
|
12
|
+
with proper stats recording and size check.
|
|
13
|
+
3. **Tool Ready** — ``pre_tool_call`` : environment readiness pre-check
|
|
14
|
+
with auto-fix and skip-retry feedback for missing dependencies.
|
|
15
|
+
4. **Command rewriting** — ``pre_tool_call`` : blocks shell commands
|
|
16
|
+
and suggests RTK-rewritten equivalents. Hermes's hook cannot modify
|
|
17
|
+
arguments, so the agent must re-execute with the suggested command
|
|
18
|
+
(one extra round-trip). Safe: ``rtk rewrite`` only does text
|
|
19
|
+
substitution, never executes the command.
|
|
20
|
+
5. **Session tracking** — ``on_session_start`` : propagates agent/session
|
|
21
|
+
IDs to tokenless stats recording.
|
|
22
|
+
|
|
23
|
+
Not available in Hermes: schema compression (Hermes hooks do not expose
|
|
24
|
+
tool schemas).
|
|
25
|
+
|
|
26
|
+
Every hook degrades gracefully: if ``tokenless`` is not installed, all
|
|
27
|
+
hooks are silently skipped.
|
|
28
|
+
|
|
29
|
+
Activation is controlled by the Hermes plugin system — list ``tokenless`` in
|
|
30
|
+
``plugins.enabled`` in ``config.yaml``, or enable via
|
|
31
|
+
``hermes plugins enable tokenless``.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import json
|
|
37
|
+
import logging
|
|
38
|
+
import os
|
|
39
|
+
import subprocess
|
|
40
|
+
import sys
|
|
41
|
+
from typing import Any
|
|
42
|
+
|
|
43
|
+
# Resolve shared hook utilities (common/hooks/) with FHS fallback paths.
|
|
44
|
+
# Primary: relative path — realpath needed because install.sh symlinks
|
|
45
|
+
# __init__.py into ~/.hermes/plugins/, and plain __file__ points to the
|
|
46
|
+
# symlink path; resolving .. from the adapter dir hits common/hooks.
|
|
47
|
+
# Fallbacks: system and user FHS paths — needed when the plugin bundle is
|
|
48
|
+
# *copied* into ~/.hermes/plugins/tokenless/ (e.g. by the anolisa driver)
|
|
49
|
+
# instead of symlinked, so the relative path resolves nowhere. User-scope
|
|
50
|
+
# candidates honor XDG_DATA_HOME (anolisa FsLayout::user prefers it over
|
|
51
|
+
# ~/.local/share).
|
|
52
|
+
#
|
|
53
|
+
# Trust model (aligned with codex/scripts/rewrite-hook, bash
|
|
54
|
+
# is_trusted_file, and Rust is_trusted_path): system FHS paths are
|
|
55
|
+
# unconditional; elsewhere the hooks directory, its parent, and the
|
|
56
|
+
# hook_utils.py file itself must be owned by the current user or root and
|
|
57
|
+
# must not be world-writable. A candidate that exists but is rejected or
|
|
58
|
+
# incomplete does not stop the search — later candidates are still tried,
|
|
59
|
+
# and every rejection reason is kept for the final diagnostic.
|
|
60
|
+
_HERE = os.path.dirname(os.path.realpath(__file__))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _validate_hooks_dir(path: str) -> str | None:
|
|
64
|
+
"""Validate a candidate hooks directory for importing hook_utils.
|
|
65
|
+
|
|
66
|
+
Returns None when the directory is trusted and contains an importable
|
|
67
|
+
hook_utils.py, otherwise a human-readable rejection reason.
|
|
68
|
+
"""
|
|
69
|
+
if not path or not os.path.isabs(path):
|
|
70
|
+
return "not an absolute path"
|
|
71
|
+
real = os.path.realpath(path)
|
|
72
|
+
if not os.path.isdir(real):
|
|
73
|
+
return "directory does not exist"
|
|
74
|
+
module = os.path.join(real, "hook_utils.py")
|
|
75
|
+
if not os.path.isfile(module):
|
|
76
|
+
return "hook_utils.py missing (incomplete or residual install)"
|
|
77
|
+
# System FHS prefixes are always trusted (checked on the realpath so a
|
|
78
|
+
# symlink pointing outside a system prefix cannot bypass the check).
|
|
79
|
+
for prefix in ("/usr/share/", "/usr/local/share/", "/usr/libexec/", "/usr/lib/anolisa/"):
|
|
80
|
+
if real.startswith(prefix):
|
|
81
|
+
return None
|
|
82
|
+
# Outside system prefixes: the hooks dir, its parent, and the module
|
|
83
|
+
# file must be owned by the current uid or root and not world-writable
|
|
84
|
+
# (mirrors bash is_trusted_file / Rust is_trusted_path).
|
|
85
|
+
uid = os.getuid()
|
|
86
|
+
for p in (real, os.path.dirname(real), module):
|
|
87
|
+
try:
|
|
88
|
+
st = os.stat(p)
|
|
89
|
+
except OSError as exc:
|
|
90
|
+
return f"stat failed for {p}: {exc}"
|
|
91
|
+
if st.st_uid != uid and st.st_uid != 0:
|
|
92
|
+
return f"{p} not owned by current user or root (uid {st.st_uid})"
|
|
93
|
+
if st.st_mode & 0o002:
|
|
94
|
+
return f"{p} is world-writable"
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _resolve_hook_utils() -> tuple[str, list[str]]:
|
|
99
|
+
"""Locate a trusted shared hooks directory and make it importable.
|
|
100
|
+
|
|
101
|
+
Returns ``(resolved_path, candidate_list)``. The resolved path is
|
|
102
|
+
inserted at the front of ``sys.path`` so the shared ``hook_utils``
|
|
103
|
+
module can be imported. Raises :exc:`ImportError` when no candidate
|
|
104
|
+
passes the trust policy.
|
|
105
|
+
"""
|
|
106
|
+
# Resolve real home from passwd DB for user-install fallback path
|
|
107
|
+
# (NOT $HOME — env-controllable).
|
|
108
|
+
try:
|
|
109
|
+
import pwd as _pwd
|
|
110
|
+
real_home = _pwd.getpwuid(os.getuid()).pw_dir
|
|
111
|
+
except (ImportError, KeyError):
|
|
112
|
+
real_home = ""
|
|
113
|
+
if not os.path.isabs(real_home):
|
|
114
|
+
real_home = ""
|
|
115
|
+
|
|
116
|
+
candidates = [
|
|
117
|
+
os.path.join(_HERE, "..", "common", "hooks"), # source-tree / symlink install
|
|
118
|
+
"/usr/share/anolisa/adapters/tokenless/common/hooks", # RPM system
|
|
119
|
+
"/usr/local/share/anolisa/adapters/tokenless/common/hooks", # manual system
|
|
120
|
+
]
|
|
121
|
+
# XDG user data dir first (anolisa FsLayout::user precedence), then the
|
|
122
|
+
# passwd-home default. XDG_DATA_HOME is env-controllable, but candidates
|
|
123
|
+
# still pass the full ownership/permission validation above.
|
|
124
|
+
xdg_data = os.environ.get("XDG_DATA_HOME", "")
|
|
125
|
+
if xdg_data and os.path.isabs(xdg_data):
|
|
126
|
+
candidates.append(
|
|
127
|
+
os.path.join(xdg_data, "anolisa", "adapters", "tokenless", "common", "hooks"))
|
|
128
|
+
if real_home:
|
|
129
|
+
candidates.append(
|
|
130
|
+
os.path.join(real_home, ".local", "share",
|
|
131
|
+
"anolisa", "adapters", "tokenless", "common", "hooks"))
|
|
132
|
+
|
|
133
|
+
rejections: list[str] = []
|
|
134
|
+
for candidate in candidates:
|
|
135
|
+
reason = _validate_hooks_dir(candidate)
|
|
136
|
+
if reason is None:
|
|
137
|
+
resolved = os.path.realpath(candidate)
|
|
138
|
+
sys.path.insert(0, resolved)
|
|
139
|
+
return resolved, candidates
|
|
140
|
+
rejections.append(f" - {candidate}: {reason}")
|
|
141
|
+
|
|
142
|
+
raise ImportError(
|
|
143
|
+
"tokenless: no trusted shared hook_utils module (common/hooks/) found.\n"
|
|
144
|
+
"Candidates checked (in order):\n" + "\n".join(rejections) + "\n"
|
|
145
|
+
"Note: a candidate may be rejected by the trust policy (ownership or "
|
|
146
|
+
"permissions) even though the path exists — see the reason next to "
|
|
147
|
+
"each path. Install the tokenless common hooks (anolisa install "
|
|
148
|
+
"tokenless) or re-run adapters/tokenless/hermes/scripts/install.sh "
|
|
149
|
+
"from a complete adapter tree."
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
_HOOK_UTILS_RESOLVED, _HOOK_UTILS_CANDIDATES = _resolve_hook_utils()
|
|
154
|
+
|
|
155
|
+
from hook_utils import (
|
|
156
|
+
_TOKENLESS_FALLBACK,
|
|
157
|
+
_TOKENLESS_LOCAL_SHARE,
|
|
158
|
+
_TOKENLESS_LOCAL_LIB,
|
|
159
|
+
_RTK_FALLBACK,
|
|
160
|
+
_RTK_LOCAL_SHARE,
|
|
161
|
+
_RTK_LOCAL_LIB,
|
|
162
|
+
resolve_binary,
|
|
163
|
+
warn as _warn_shared,
|
|
164
|
+
try_parse_json as _try_parse_json,
|
|
165
|
+
is_skill_file as _is_skill_file,
|
|
166
|
+
write_context as _write_context,
|
|
167
|
+
run as _run,
|
|
168
|
+
parse_version as _parse_version,
|
|
169
|
+
SKIP_TOOLS as _SKIP_TOOLS_SHARED,
|
|
170
|
+
SHELL_TOOLS as _SHELL_TOOLS_SHARED,
|
|
171
|
+
get_thresholds,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
logger = logging.getLogger(__name__)
|
|
175
|
+
|
|
176
|
+
# ---------------------------------------------------------------------------
|
|
177
|
+
# Constants
|
|
178
|
+
# ---------------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
AGENT_ID = "hermes-agent"
|
|
181
|
+
_MIN_RESPONSE_LEN = 200
|
|
182
|
+
|
|
183
|
+
_SKIP_TOOLS: set[str] = _SKIP_TOOLS_SHARED | {
|
|
184
|
+
"session_search", "list_sessions",
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
# Use shared SHELL_TOOLS directly - all tools (including "terminal") are now
|
|
188
|
+
# defined in the unified tool_categories.json
|
|
189
|
+
_SHELL_TOOLS: set[str] = _SHELL_TOOLS_SHARED
|
|
190
|
+
|
|
191
|
+
_MIN_RTK_VERSION = (0, 35, 0)
|
|
192
|
+
|
|
193
|
+
# ---------------------------------------------------------------------------
|
|
194
|
+
# Binary resolution (thin wrapper over shared cached resolve_binary)
|
|
195
|
+
# ---------------------------------------------------------------------------
|
|
196
|
+
|
|
197
|
+
# Hermes-specific fallback paths for the RTK binary.
|
|
198
|
+
_RTK_LIB_FALLBACK = "/usr/lib/anolisa/tokenless/rtk"
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _resolve_binary(name: str, fallback: str) -> str | None:
|
|
202
|
+
"""Resolve binary with hermes-specific fallback paths (cached via shared)."""
|
|
203
|
+
local_bin = os.path.join(os.path.expanduser("~"), ".local", "bin", name)
|
|
204
|
+
if name == "rtk":
|
|
205
|
+
return resolve_binary(name, fallback, _RTK_LIB_FALLBACK, local_bin, _RTK_LOCAL_LIB, _RTK_LOCAL_SHARE)
|
|
206
|
+
return resolve_binary(name, fallback, local_bin, _TOKENLESS_LOCAL_LIB, _TOKENLESS_LOCAL_SHARE)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _have(name: str, fallback: str) -> bool:
|
|
210
|
+
return _resolve_binary(name, fallback) is not None
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
# ---------------------------------------------------------------------------
|
|
214
|
+
# Helpers (shared via hook_utils)
|
|
215
|
+
# ---------------------------------------------------------------------------
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
# ---------------------------------------------------------------------------
|
|
219
|
+
# 1. Response Compression (via tokenless compress-response)
|
|
220
|
+
# ---------------------------------------------------------------------------
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _compress_response(
|
|
224
|
+
tool_name: str,
|
|
225
|
+
result: str,
|
|
226
|
+
session_id: str,
|
|
227
|
+
tool_call_id: str,
|
|
228
|
+
) -> str | None:
|
|
229
|
+
tokenless_bin = _resolve_binary("tokenless", _TOKENLESS_FALLBACK)
|
|
230
|
+
if not tokenless_bin:
|
|
231
|
+
return None
|
|
232
|
+
|
|
233
|
+
parsed = _try_parse_json(result)
|
|
234
|
+
if not isinstance(parsed, (dict, list)):
|
|
235
|
+
return None
|
|
236
|
+
|
|
237
|
+
# 3-layer dispatch: shell tools use moderate truncation, API tools use zero-truncation
|
|
238
|
+
thresholds = get_thresholds(tool_name)
|
|
239
|
+
|
|
240
|
+
cmd = [
|
|
241
|
+
tokenless_bin, "compress-response",
|
|
242
|
+
"--agent-id", AGENT_ID,
|
|
243
|
+
"--truncate-strings-at", str(thresholds[0]),
|
|
244
|
+
"--truncate-arrays-at", str(thresholds[1]),
|
|
245
|
+
"--max-depth", str(thresholds[2]),
|
|
246
|
+
]
|
|
247
|
+
if session_id:
|
|
248
|
+
cmd.extend(["--session-id", session_id])
|
|
249
|
+
if tool_call_id:
|
|
250
|
+
cmd.extend(["--tool-use-id", tool_call_id])
|
|
251
|
+
|
|
252
|
+
proc = _run(cmd, result)
|
|
253
|
+
if not proc or proc.returncode != 0 or not proc.stdout.strip():
|
|
254
|
+
return None
|
|
255
|
+
|
|
256
|
+
compressed = proc.stdout.strip()
|
|
257
|
+
if len(compressed) >= len(result):
|
|
258
|
+
return None
|
|
259
|
+
return compressed
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
# ---------------------------------------------------------------------------
|
|
263
|
+
# 2. TOON Encoding (via tokenless compress-toon)
|
|
264
|
+
# ---------------------------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _encode_toon(data: str, session_id: str = "", tool_call_id: str = "") -> tuple[str, int] | None:
|
|
268
|
+
tokenless_bin = _resolve_binary("tokenless", _TOKENLESS_FALLBACK)
|
|
269
|
+
if not tokenless_bin:
|
|
270
|
+
return None
|
|
271
|
+
|
|
272
|
+
parsed = _try_parse_json(data)
|
|
273
|
+
if not isinstance(parsed, (dict, list)):
|
|
274
|
+
return None
|
|
275
|
+
|
|
276
|
+
cmd = [tokenless_bin, "compress-toon", "--agent-id", AGENT_ID]
|
|
277
|
+
if session_id:
|
|
278
|
+
cmd.extend(["--session-id", session_id])
|
|
279
|
+
if tool_call_id:
|
|
280
|
+
cmd.extend(["--tool-use-id", tool_call_id])
|
|
281
|
+
|
|
282
|
+
proc = _run(cmd, data, timeout=1)
|
|
283
|
+
if not proc or proc.returncode != 0 or not proc.stdout.strip():
|
|
284
|
+
return None
|
|
285
|
+
|
|
286
|
+
toon_text = proc.stdout.strip()
|
|
287
|
+
if len(toon_text) >= len(data):
|
|
288
|
+
return None
|
|
289
|
+
|
|
290
|
+
savings_pct = 0
|
|
291
|
+
if len(data) > 0:
|
|
292
|
+
savings_pct = (len(data) - len(toon_text)) * 100 // len(data)
|
|
293
|
+
|
|
294
|
+
return toon_text, savings_pct
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
# ---------------------------------------------------------------------------
|
|
298
|
+
# 3. Tool Ready (via tokenless env-check)
|
|
299
|
+
# ---------------------------------------------------------------------------
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _env_check(tool_name: str) -> str | None:
|
|
303
|
+
"""Run tool-ready env-check and return feedback if tool is not ready."""
|
|
304
|
+
tokenless_bin = _resolve_binary("tokenless", _TOKENLESS_FALLBACK)
|
|
305
|
+
if not tokenless_bin:
|
|
306
|
+
return None
|
|
307
|
+
|
|
308
|
+
proc = _run([tokenless_bin, "env-check", "--tool", tool_name, "--json"], "", timeout=5)
|
|
309
|
+
if not proc or not proc.stdout.strip():
|
|
310
|
+
return None
|
|
311
|
+
|
|
312
|
+
try:
|
|
313
|
+
parsed = json.loads(proc.stdout)
|
|
314
|
+
except json.JSONDecodeError:
|
|
315
|
+
return None
|
|
316
|
+
|
|
317
|
+
status = parsed.get("status", "UNKNOWN")
|
|
318
|
+
if status in ("UNKNOWN", "READY"):
|
|
319
|
+
return None
|
|
320
|
+
|
|
321
|
+
# Attempt auto-fix
|
|
322
|
+
proc = _run([tokenless_bin, "env-check", "--tool", tool_name, "--fix", "--json"], "", timeout=10)
|
|
323
|
+
if not proc or not proc.stdout.strip():
|
|
324
|
+
return _not_ready_msg(tool_name)
|
|
325
|
+
|
|
326
|
+
try:
|
|
327
|
+
fix_parsed = json.loads(proc.stdout)
|
|
328
|
+
except json.JSONDecodeError:
|
|
329
|
+
return _not_ready_msg(tool_name)
|
|
330
|
+
|
|
331
|
+
if fix_parsed.get("status") == "READY":
|
|
332
|
+
return None
|
|
333
|
+
|
|
334
|
+
diagnostic = fix_parsed.get("diagnostic", "")
|
|
335
|
+
return diagnostic or _not_ready_msg(tool_name)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _not_ready_msg(tool_name: str) -> str:
|
|
339
|
+
return f"[tokenless:ready] {tool_name}: NOT_READY. Skip retry."
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
# ---------------------------------------------------------------------------
|
|
343
|
+
# 4. Command Rewriting (via rtk rewrite)
|
|
344
|
+
# ---------------------------------------------------------------------------
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _try_rewrite(
|
|
348
|
+
args: Any,
|
|
349
|
+
session_id: str,
|
|
350
|
+
tool_call_id: str,
|
|
351
|
+
) -> dict[str, str] | None:
|
|
352
|
+
"""Attempt RTK command rewrite for terminal tool calls.
|
|
353
|
+
|
|
354
|
+
Calls ``rtk rewrite <command>`` — a pure text substitution that never
|
|
355
|
+
executes the command. On success, returns a block directive suggesting
|
|
356
|
+
the rewritten command so the agent re-executes with the optimized version.
|
|
357
|
+
"""
|
|
358
|
+
rtk_bin = _resolve_binary("rtk", _RTK_FALLBACK)
|
|
359
|
+
if not rtk_bin:
|
|
360
|
+
return None
|
|
361
|
+
|
|
362
|
+
if not isinstance(args, dict):
|
|
363
|
+
return None
|
|
364
|
+
|
|
365
|
+
command = args.get("command", "")
|
|
366
|
+
if not command:
|
|
367
|
+
return None
|
|
368
|
+
|
|
369
|
+
# Version guard — non-fatal
|
|
370
|
+
try:
|
|
371
|
+
ver_proc = subprocess.run(
|
|
372
|
+
[rtk_bin, "--version"], capture_output=True, text=True, timeout=3,
|
|
373
|
+
)
|
|
374
|
+
ver = _parse_version(ver_proc.stdout)
|
|
375
|
+
if ver and ver < _MIN_RTK_VERSION:
|
|
376
|
+
logger.warning("tokenless: rtk %s too old (need >= 0.35.0), rewrite skipped", ver_proc.stdout.strip())
|
|
377
|
+
return None
|
|
378
|
+
except Exception:
|
|
379
|
+
pass
|
|
380
|
+
|
|
381
|
+
# Write context file so rtk (running as proxy later) can recover IDs
|
|
382
|
+
_write_context(AGENT_ID, session_id, tool_call_id)
|
|
383
|
+
|
|
384
|
+
# Set env vars for rtk stats context
|
|
385
|
+
env = os.environ.copy()
|
|
386
|
+
env["TOKENLESS_AGENT_ID"] = AGENT_ID
|
|
387
|
+
if session_id:
|
|
388
|
+
env["TOKENLESS_SESSION_ID"] = session_id
|
|
389
|
+
if tool_call_id:
|
|
390
|
+
env["TOKENLESS_TOOL_USE_ID"] = tool_call_id
|
|
391
|
+
|
|
392
|
+
proc = subprocess.run(
|
|
393
|
+
[rtk_bin, "rewrite", command],
|
|
394
|
+
capture_output=True, text=True, timeout=5, env=env,
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
# Exit code protocol (from rtk rewrite_cmd.rs):
|
|
398
|
+
# 0 = rewrite available, Allow verdict (auto-allow by permission rule)
|
|
399
|
+
# 1 = no RTK equivalent (passthrough)
|
|
400
|
+
# 2 = deny rule matched (let Hermes handle)
|
|
401
|
+
# 3 = Ask/Default verdict (rewrite available but permission model requires
|
|
402
|
+
# user confirmation; in non-interactive hook context, treat as valid
|
|
403
|
+
# rewrite since the intent is token optimization, not permission gating)
|
|
404
|
+
if proc.returncode == 1 or proc.returncode == 2:
|
|
405
|
+
return None
|
|
406
|
+
if proc.returncode != 0 and proc.returncode != 3:
|
|
407
|
+
return None
|
|
408
|
+
|
|
409
|
+
rewritten = proc.stdout.strip()
|
|
410
|
+
if not rewritten or rewritten == command:
|
|
411
|
+
return None
|
|
412
|
+
|
|
413
|
+
logger.info("tokenless: rtk rewrite %s → %s", command, rewritten)
|
|
414
|
+
return {
|
|
415
|
+
"action": "block",
|
|
416
|
+
"message": f"[tokenless:rewrite] Re-execute as: {rewritten}",
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
# ---------------------------------------------------------------------------
|
|
421
|
+
# Hook callbacks
|
|
422
|
+
# ---------------------------------------------------------------------------
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def on_session_start(**kwargs: Any) -> None:
|
|
426
|
+
"""Record session mapping for stats context."""
|
|
427
|
+
session_id = kwargs.get("session_id", "")
|
|
428
|
+
if session_id:
|
|
429
|
+
os.environ["TOKENLESS_SESSION_ID"] = str(session_id)
|
|
430
|
+
logger.debug("tokenless: session_start session_id=%s", session_id)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def on_pre_tool_call(
|
|
434
|
+
tool_name: str = "",
|
|
435
|
+
args: Any = None,
|
|
436
|
+
task_id: str = "",
|
|
437
|
+
session_id: str = "",
|
|
438
|
+
tool_call_id: str = "",
|
|
439
|
+
**kwargs: Any,
|
|
440
|
+
) -> dict[str, str] | None:
|
|
441
|
+
"""Tool Ready + RTK rewrite pre-check.
|
|
442
|
+
|
|
443
|
+
Step 1: env-check blocks when the tool's environment is not ready.
|
|
444
|
+
Step 2: for ``terminal`` calls, blocks and suggests RTK-rewritten
|
|
445
|
+
command (one extra round-trip; safe — rtk rewrite never executes).
|
|
446
|
+
"""
|
|
447
|
+
# Step 1: env-check (all tools, needs tokenless)
|
|
448
|
+
if _have("tokenless", _TOKENLESS_FALLBACK):
|
|
449
|
+
if session_id:
|
|
450
|
+
os.environ["TOKENLESS_SESSION_ID"] = str(session_id)
|
|
451
|
+
feedback = _env_check(tool_name)
|
|
452
|
+
if feedback:
|
|
453
|
+
logger.info("tokenless: tool-ready blocking %s — %s", tool_name, feedback)
|
|
454
|
+
return {"action": "block", "message": feedback}
|
|
455
|
+
|
|
456
|
+
# Step 2: RTK rewrite (terminal only, needs rtk)
|
|
457
|
+
if tool_name in _SHELL_TOOLS and _have("rtk", _RTK_FALLBACK):
|
|
458
|
+
result = _try_rewrite(args, str(session_id), str(tool_call_id))
|
|
459
|
+
if result:
|
|
460
|
+
return result
|
|
461
|
+
|
|
462
|
+
return None
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def on_transform_tool_result(
|
|
466
|
+
tool_name: str = "",
|
|
467
|
+
args: Any = None,
|
|
468
|
+
result: str = "",
|
|
469
|
+
task_id: str = "",
|
|
470
|
+
session_id: str = "",
|
|
471
|
+
tool_call_id: str = "",
|
|
472
|
+
duration_ms: int = 0,
|
|
473
|
+
**kwargs: Any,
|
|
474
|
+
) -> str | None:
|
|
475
|
+
"""Response compression + TOON encoding pipeline.
|
|
476
|
+
|
|
477
|
+
Replaces the tool result string with a compressed/TOON-encoded version.
|
|
478
|
+
Runs after post_tool_call; first valid string return wins.
|
|
479
|
+
|
|
480
|
+
Content retrieval tools (Read/Glob/Grep) are skipped entirely.
|
|
481
|
+
Shell/exec tools (Bash/Shell) use moderate truncation (64K/128/8).
|
|
482
|
+
All other tools use zero-truncation compress-response + TOON.
|
|
483
|
+
"""
|
|
484
|
+
if not _have("tokenless", _TOKENLESS_FALLBACK):
|
|
485
|
+
return None
|
|
486
|
+
|
|
487
|
+
# Content retrieval — skip entirely (preserve integrity)
|
|
488
|
+
if tool_name in _SKIP_TOOLS:
|
|
489
|
+
return None
|
|
490
|
+
|
|
491
|
+
if not result or result in ("{}", "[]"):
|
|
492
|
+
return None
|
|
493
|
+
|
|
494
|
+
# Skip skill files (YAML frontmatter)
|
|
495
|
+
if _is_skill_file(result):
|
|
496
|
+
return None
|
|
497
|
+
|
|
498
|
+
# Skip small responses
|
|
499
|
+
if len(result) < _MIN_RESPONSE_LEN:
|
|
500
|
+
return None
|
|
501
|
+
|
|
502
|
+
# Validate it's JSON
|
|
503
|
+
parsed = _try_parse_json(result)
|
|
504
|
+
if parsed is None:
|
|
505
|
+
return None
|
|
506
|
+
|
|
507
|
+
# Normalize: result is already a JSON string (Hermes tool contract)
|
|
508
|
+
original = result
|
|
509
|
+
original_len = len(original)
|
|
510
|
+
|
|
511
|
+
# Step 1: Response compression (per-layer thresholds via get_thresholds)
|
|
512
|
+
compressed = _compress_response(tool_name, result,
|
|
513
|
+
str(session_id), str(tool_call_id))
|
|
514
|
+
current = compressed if compressed else result
|
|
515
|
+
|
|
516
|
+
# Step 2: TOON encoding
|
|
517
|
+
toon_result = _encode_toon(current, str(session_id), str(tool_call_id))
|
|
518
|
+
used_compression = compressed is not None
|
|
519
|
+
used_toon = toon_result is not None
|
|
520
|
+
|
|
521
|
+
if not used_compression and not used_toon:
|
|
522
|
+
return None
|
|
523
|
+
|
|
524
|
+
# Build final output
|
|
525
|
+
if used_toon:
|
|
526
|
+
toon_text, savings_pct = toon_result
|
|
527
|
+
final = toon_text
|
|
528
|
+
final_len = len(toon_text)
|
|
529
|
+
savings_label = (
|
|
530
|
+
"response compressed + TOON encoded"
|
|
531
|
+
if used_compression
|
|
532
|
+
else "TOON encoded"
|
|
533
|
+
)
|
|
534
|
+
else:
|
|
535
|
+
final = current # type: ignore[assignment]
|
|
536
|
+
final_len = len(final)
|
|
537
|
+
savings_pct = (original_len - final_len) * 100 // original_len if original_len else 0
|
|
538
|
+
savings_label = "response compressed"
|
|
539
|
+
|
|
540
|
+
logger.info(
|
|
541
|
+
"tokenless: %s %s: %d -> %d chars (%d%% reduction)",
|
|
542
|
+
savings_label, tool_name, original_len, final_len, savings_pct,
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
return final
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
# ---------------------------------------------------------------------------
|
|
549
|
+
# Plugin entry point
|
|
550
|
+
# ---------------------------------------------------------------------------
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def register(ctx: Any) -> None:
|
|
554
|
+
"""Register all tokenless hooks with the Hermes plugin system."""
|
|
555
|
+
|
|
556
|
+
ctx.register_hook("on_session_start", on_session_start)
|
|
557
|
+
ctx.register_hook("pre_tool_call", on_pre_tool_call)
|
|
558
|
+
ctx.register_hook("transform_tool_result", on_transform_tool_result)
|
|
559
|
+
|
|
560
|
+
# Log what's active
|
|
561
|
+
features: list[str] = []
|
|
562
|
+
if _have("tokenless", _TOKENLESS_FALLBACK):
|
|
563
|
+
features.append("response-compression")
|
|
564
|
+
features.append("toon-encoding")
|
|
565
|
+
features.append("tool-ready")
|
|
566
|
+
if _have("rtk", _RTK_FALLBACK):
|
|
567
|
+
features.append("rtk-rewrite")
|
|
568
|
+
|
|
569
|
+
logger.info(
|
|
570
|
+
"tokenless: Hermes plugin registered — active features: %s",
|
|
571
|
+
", ".join(features) if features else "none (install tokenless/rtk binary)",
|
|
572
|
+
)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
name: tokenless
|
|
2
|
+
version: "0.7.7"
|
|
3
|
+
description: "Token-Less context compression for Hermes Agent — response compression, TOON encoding, command rewriting, and registered but hard-disabled Tool Ready"
|
|
4
|
+
author: ANOLISA
|
|
5
|
+
requires_env: []
|
|
6
|
+
provides_hooks:
|
|
7
|
+
- transform_tool_result
|
|
8
|
+
- pre_tool_call
|
|
9
|
+
- on_session_start
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# detect.sh — Inspect tokenless Hermes integration. Read-only.
|
|
3
|
+
#
|
|
4
|
+
# Reports hermes CLI, Hermes home, tokenless plugin install state, runtime
|
|
5
|
+
# binary, and adapter resource availability. Exit codes:
|
|
6
|
+
# 0 = installed and ready
|
|
7
|
+
# 1 = not installed but installable
|
|
8
|
+
# 2 = missing prerequisites
|
|
9
|
+
set -euo pipefail
|
|
10
|
+
|
|
11
|
+
AGENT="${ANOLISA_TARGET:-hermes}"
|
|
12
|
+
COMPONENT="${ANOLISA_COMPONENT:-tokenless}"
|
|
13
|
+
ADAPTER_DIR="${ANOLISA_ADAPTER_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}"
|
|
14
|
+
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
|
15
|
+
HERMES_BIN="${HERMES_BIN:-}"
|
|
16
|
+
export PATH="$HOME/.local/bin:${HERMES_HOME%/}/bin:/usr/local/bin:$PATH"
|
|
17
|
+
|
|
18
|
+
PLUGIN_ID="tokenless"
|
|
19
|
+
PLUGIN_SRC="$ADAPTER_DIR/hermes"
|
|
20
|
+
|
|
21
|
+
line() { printf '[%s] %s\n' "$COMPONENT" "$*"; }
|
|
22
|
+
field() { printf '[%s] %-26s %s\n' "$COMPONENT" "$1" "$2"; }
|
|
23
|
+
|
|
24
|
+
PREREQ_MISSING=()
|
|
25
|
+
INSTALL_MISSING=()
|
|
26
|
+
note_prereq_missing() { PREREQ_MISSING+=("$1"); }
|
|
27
|
+
note_install_missing() { INSTALL_MISSING+=("$1"); }
|
|
28
|
+
|
|
29
|
+
if [ -z "$HERMES_BIN" ]; then
|
|
30
|
+
HERMES_BIN="$(command -v hermes 2>/dev/null || true)"
|
|
31
|
+
fi
|
|
32
|
+
|
|
33
|
+
line "${AGENT} detect"
|
|
34
|
+
if [ -n "$HERMES_BIN" ] && [ -x "$HERMES_BIN" ]; then
|
|
35
|
+
field "hermes CLI" "present (${HERMES_BIN})"
|
|
36
|
+
else
|
|
37
|
+
field "hermes CLI" "missing"
|
|
38
|
+
note_prereq_missing "hermes CLI"
|
|
39
|
+
fi
|
|
40
|
+
|
|
41
|
+
if [ -d "$HERMES_HOME" ]; then
|
|
42
|
+
field "hermes home" "present (${HERMES_HOME})"
|
|
43
|
+
else
|
|
44
|
+
field "hermes home" "not installed (${HERMES_HOME})"
|
|
45
|
+
note_install_missing "hermes home"
|
|
46
|
+
fi
|
|
47
|
+
|
|
48
|
+
runtime_bin="$(command -v tokenless 2>/dev/null || true)"
|
|
49
|
+
if [ -n "$runtime_bin" ]; then
|
|
50
|
+
field "tokenless binary" "present (${runtime_bin})"
|
|
51
|
+
else
|
|
52
|
+
field "tokenless binary" "missing"
|
|
53
|
+
note_prereq_missing "tokenless binary"
|
|
54
|
+
fi
|
|
55
|
+
|
|
56
|
+
if [ -d "$PLUGIN_SRC" ] && [ -f "$PLUGIN_SRC/plugin.yaml" ] && [ -f "$PLUGIN_SRC/__init__.py" ]; then
|
|
57
|
+
field "plugin resource" "present (${PLUGIN_SRC})"
|
|
58
|
+
else
|
|
59
|
+
field "plugin resource" "missing (${PLUGIN_SRC})"
|
|
60
|
+
note_prereq_missing "plugin resource"
|
|
61
|
+
fi
|
|
62
|
+
|
|
63
|
+
plugin_dst="${HERMES_HOME%/}/plugins/${PLUGIN_ID}"
|
|
64
|
+
if [ -d "$plugin_dst" ] || [ -L "$plugin_dst" ]; then
|
|
65
|
+
field "${PLUGIN_ID} plugin" "installed (${plugin_dst})"
|
|
66
|
+
else
|
|
67
|
+
field "${PLUGIN_ID} plugin" "missing (${plugin_dst})"
|
|
68
|
+
note_install_missing "${PLUGIN_ID} plugin"
|
|
69
|
+
fi
|
|
70
|
+
|
|
71
|
+
if [ ${#PREREQ_MISSING[@]} -gt 0 ]; then
|
|
72
|
+
line "${AGENT}: missing prerequisites (${PREREQ_MISSING[*]})"
|
|
73
|
+
exit 2
|
|
74
|
+
fi
|
|
75
|
+
if [ ${#INSTALL_MISSING[@]} -gt 0 ]; then
|
|
76
|
+
line "${AGENT}: not installed (ready to install)"
|
|
77
|
+
exit 1
|
|
78
|
+
fi
|
|
79
|
+
line "${AGENT}: ready"
|
|
80
|
+
exit 0
|