anolisa-tokenless 0.7.13 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +219 -87
- package/adapters/tokenless/claude-code/.claude-plugin/plugin.json +1 -1
- package/adapters/tokenless/claude-code/hooks/run-hook.sh +62 -0
- package/adapters/tokenless/codex/.codex-plugin/plugin.json +5 -4
- package/adapters/tokenless/codex/README.md +22 -32
- package/adapters/tokenless/codex/hooks/hooks.json +3 -3
- package/adapters/tokenless/codex/scripts/response-diagnostics +170 -0
- package/adapters/tokenless/common/cosh-extension.json +4 -4
- package/adapters/tokenless/common/hooks/compress_response_hook.py +250 -307
- package/adapters/tokenless/common/hooks/compress_schema_hook.py +34 -42
- package/adapters/tokenless/common/hooks/hook_utils.py +281 -44
- package/adapters/tokenless/common/hooks/rewrite_hook.py +53 -169
- package/adapters/tokenless/dsh/dist/index.js +353 -264
- package/adapters/tokenless/dsh/package.json +2 -2
- package/adapters/tokenless/hermes/__init__.py +191 -355
- package/adapters/tokenless/hermes/plugin.yaml +2 -2
- package/adapters/tokenless/manifest.json +18 -3
- package/adapters/tokenless/openclaw/dist/index.d.ts +4 -16
- package/adapters/tokenless/openclaw/dist/index.js +291 -507
- package/adapters/tokenless/openclaw/index.ts +408 -628
- package/adapters/tokenless/openclaw/openclaw.plugin.json +4 -20
- package/adapters/tokenless/openclaw/package.json +6 -4
- package/adapters/tokenless/qoder/.qoder-plugin/plugin.json +1 -1
- package/adapters/tokenless/qwencode/hooks/run-hook.sh +62 -0
- package/adapters/tokenless/qwencode/qwen-extension.json +4 -4
- package/adapters/tokenless/qwenpaw/plugin.json +17 -0
- package/adapters/tokenless/qwenpaw/plugin.py +390 -0
- package/adapters/tokenless/qwenpaw/requirements.txt +6 -0
- package/adapters/tokenless/qwenpaw/scripts/detect.sh +131 -0
- package/adapters/tokenless/qwenpaw/scripts/install.sh +98 -0
- package/adapters/tokenless/qwenpaw/scripts/uninstall.sh +61 -0
- package/package.json +5 -5
- package/adapters/tokenless/codex/scripts/compress-response +0 -445
- package/adapters/tokenless/common/hooks/compress_toon_hook.py +0 -171
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anolisa/dsh-tokenless",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Native DeepSeek Harness
|
|
3
|
+
"version": "0.8.0",
|
|
4
|
+
"description": "Native DeepSeek Harness lifecycle integration for Tokenless",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"exports": {
|
|
@@ -1,30 +1,11 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
2. **TOON encoding** — ``transform_tool_result`` : pipeline step after
|
|
10
|
-
response compression; re-encodes JSON results to TOON format via
|
|
11
|
-
``tokenless compress-toon`` for additional token savings (15-40%)
|
|
12
|
-
with proper stats recording and size check.
|
|
13
|
-
3. **Tool Ready** — ``pre_tool_call`` : environment readiness pre-check
|
|
14
|
-
with auto-fix and skip-retry feedback for missing dependencies.
|
|
15
|
-
4. **Command rewriting** — ``pre_tool_call`` : blocks shell commands
|
|
16
|
-
and suggests RTK-rewritten equivalents. Hermes's hook cannot modify
|
|
17
|
-
arguments, so the agent must re-execute with the suggested command
|
|
18
|
-
(one extra round-trip). Safe: ``rtk rewrite`` only does text
|
|
19
|
-
substitution, never executes the command.
|
|
20
|
-
5. **Session tracking** — ``on_session_start`` : propagates agent/session
|
|
21
|
-
IDs to tokenless stats recording.
|
|
22
|
-
|
|
23
|
-
Not available in Hermes: schema compression (Hermes hooks do not expose
|
|
24
|
-
tool schemas).
|
|
25
|
-
|
|
26
|
-
Every hook degrades gracefully: if ``tokenless`` is not installed, all
|
|
27
|
-
hooks are silently skipped.
|
|
1
|
+
"""Tokenless lifecycle adapter for Hermes Agent.
|
|
2
|
+
|
|
3
|
+
Hermes cannot replace tool arguments on older supported releases, so PreTool
|
|
4
|
+
blocks a shell call and suggests the Core-rewritten command. PostTool sends the
|
|
5
|
+
final model-bound result to Core and applies only the returned disposition.
|
|
6
|
+
Schema compression is not available from the Hermes hook surface. Marker-directed
|
|
7
|
+
recovery uses Hermes's existing shell tool and the trusted local Tokenless CLI.
|
|
8
|
+
Tool Ready remains product-wide hard-disabled.
|
|
28
9
|
|
|
29
10
|
Activation is controlled by the Hermes plugin system — list ``tokenless`` in
|
|
30
11
|
``plugins.enabled`` in ``config.yaml``, or enable via
|
|
@@ -36,7 +17,7 @@ from __future__ import annotations
|
|
|
36
17
|
import json
|
|
37
18
|
import logging
|
|
38
19
|
import os
|
|
39
|
-
import
|
|
20
|
+
import shlex
|
|
40
21
|
import sys
|
|
41
22
|
from typing import Any
|
|
42
23
|
|
|
@@ -107,6 +88,7 @@ def _resolve_hook_utils() -> tuple[str, list[str]]:
|
|
|
107
88
|
# (NOT $HOME — env-controllable).
|
|
108
89
|
try:
|
|
109
90
|
import pwd as _pwd
|
|
91
|
+
|
|
110
92
|
real_home = _pwd.getpwuid(os.getuid()).pw_dir
|
|
111
93
|
except (ImportError, KeyError):
|
|
112
94
|
real_home = ""
|
|
@@ -114,9 +96,10 @@ def _resolve_hook_utils() -> tuple[str, list[str]]:
|
|
|
114
96
|
real_home = ""
|
|
115
97
|
|
|
116
98
|
candidates = [
|
|
117
|
-
|
|
118
|
-
"
|
|
119
|
-
"/usr/
|
|
99
|
+
# Source-tree / symlink install.
|
|
100
|
+
os.path.join(_HERE, "..", "common", "hooks"),
|
|
101
|
+
"/usr/share/anolisa/adapters/tokenless/common/hooks", # RPM system
|
|
102
|
+
"/usr/local/share/anolisa/adapters/tokenless/common/hooks", # Manual system
|
|
120
103
|
]
|
|
121
104
|
# XDG user data dir first (anolisa FsLayout::user precedence), then the
|
|
122
105
|
# passwd-home default. XDG_DATA_HOME is env-controllable, but candidates
|
|
@@ -124,11 +107,14 @@ def _resolve_hook_utils() -> tuple[str, list[str]]:
|
|
|
124
107
|
xdg_data = os.environ.get("XDG_DATA_HOME", "")
|
|
125
108
|
if xdg_data and os.path.isabs(xdg_data):
|
|
126
109
|
candidates.append(
|
|
127
|
-
os.path.join(xdg_data, "anolisa", "adapters", "tokenless", "common", "hooks")
|
|
110
|
+
os.path.join(xdg_data, "anolisa", "adapters", "tokenless", "common", "hooks")
|
|
111
|
+
)
|
|
128
112
|
if real_home:
|
|
129
113
|
candidates.append(
|
|
130
|
-
os.path.join(
|
|
131
|
-
|
|
114
|
+
os.path.join(
|
|
115
|
+
real_home, ".local", "share", "anolisa", "adapters", "tokenless", "common", "hooks"
|
|
116
|
+
)
|
|
117
|
+
)
|
|
132
118
|
|
|
133
119
|
rejections: list[str] = []
|
|
134
120
|
for candidate in candidates:
|
|
@@ -153,22 +139,22 @@ def _resolve_hook_utils() -> tuple[str, list[str]]:
|
|
|
153
139
|
_HOOK_UTILS_RESOLVED, _HOOK_UTILS_CANDIDATES = _resolve_hook_utils()
|
|
154
140
|
|
|
155
141
|
from hook_utils import (
|
|
156
|
-
_TOKENLESS_FALLBACK,
|
|
157
|
-
_TOKENLESS_LOCAL_SHARE,
|
|
158
|
-
_TOKENLESS_LOCAL_LIB,
|
|
159
142
|
_RTK_FALLBACK,
|
|
160
|
-
_RTK_LOCAL_SHARE,
|
|
161
143
|
_RTK_LOCAL_LIB,
|
|
144
|
+
_RTK_LOCAL_SHARE,
|
|
145
|
+
_TOKENLESS_FALLBACK,
|
|
146
|
+
_TOKENLESS_LOCAL_LIB,
|
|
147
|
+
_TOKENLESS_LOCAL_SHARE,
|
|
148
|
+
)
|
|
149
|
+
from hook_utils import SHELL_TOOLS as _SHELL_TOOLS_SHARED
|
|
150
|
+
from hook_utils import SKIP_TOOLS as _SKIP_TOOLS_SHARED
|
|
151
|
+
from hook_utils import (
|
|
152
|
+
build_post_tool_request,
|
|
153
|
+
build_pre_tool_request,
|
|
154
|
+
is_tokenless_retrieve_command,
|
|
162
155
|
resolve_binary,
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
is_skill_file as _is_skill_file,
|
|
166
|
-
write_context as _write_context,
|
|
167
|
-
run as _run,
|
|
168
|
-
parse_version as _parse_version,
|
|
169
|
-
SKIP_TOOLS as _SKIP_TOOLS_SHARED,
|
|
170
|
-
SHELL_TOOLS as _SHELL_TOOLS_SHARED,
|
|
171
|
-
get_thresholds,
|
|
156
|
+
run_compress,
|
|
157
|
+
tokenless_retrieve_command_available,
|
|
172
158
|
)
|
|
173
159
|
|
|
174
160
|
logger = logging.getLogger(__name__)
|
|
@@ -178,24 +164,17 @@ logger = logging.getLogger(__name__)
|
|
|
178
164
|
# ---------------------------------------------------------------------------
|
|
179
165
|
|
|
180
166
|
AGENT_ID = "hermes-agent"
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
# Minimum payload size for the TOON encoding step. TOON on small JSON
|
|
184
|
-
# saves only a few characters (observed ~0.3% below ~500 chars) while
|
|
185
|
-
# the per-event encode cost stays the same, so payloads under this
|
|
186
|
-
# threshold keep the response-compressed form and skip TOON entirely.
|
|
187
|
-
_MIN_TOON_CHARS = 500
|
|
167
|
+
_COMPRESS_TIMEOUT_SECONDS = 8
|
|
188
168
|
|
|
189
169
|
_SKIP_TOOLS: set[str] = _SKIP_TOOLS_SHARED | {
|
|
190
|
-
"session_search",
|
|
170
|
+
"session_search",
|
|
171
|
+
"list_sessions",
|
|
191
172
|
}
|
|
192
173
|
|
|
193
174
|
# Use shared SHELL_TOOLS directly - all tools (including "terminal") are now
|
|
194
175
|
# defined in the unified tool_categories.json
|
|
195
176
|
_SHELL_TOOLS: set[str] = _SHELL_TOOLS_SHARED
|
|
196
177
|
|
|
197
|
-
_MIN_RTK_VERSION = (0, 35, 0)
|
|
198
|
-
|
|
199
178
|
# ---------------------------------------------------------------------------
|
|
200
179
|
# Binary resolution (thin wrapper over shared cached resolve_binary)
|
|
201
180
|
# ---------------------------------------------------------------------------
|
|
@@ -208,7 +187,14 @@ def _resolve_binary(name: str, fallback: str) -> str | None:
|
|
|
208
187
|
"""Resolve binary with hermes-specific fallback paths (cached via shared)."""
|
|
209
188
|
local_bin = os.path.join(os.path.expanduser("~"), ".local", "bin", name)
|
|
210
189
|
if name == "rtk":
|
|
211
|
-
return resolve_binary(
|
|
190
|
+
return resolve_binary(
|
|
191
|
+
name,
|
|
192
|
+
fallback,
|
|
193
|
+
_RTK_LIB_FALLBACK,
|
|
194
|
+
local_bin,
|
|
195
|
+
_RTK_LOCAL_LIB,
|
|
196
|
+
_RTK_LOCAL_SHARE,
|
|
197
|
+
)
|
|
212
198
|
return resolve_binary(name, fallback, local_bin, _TOKENLESS_LOCAL_LIB, _TOKENLESS_LOCAL_SHARE)
|
|
213
199
|
|
|
214
200
|
|
|
@@ -216,211 +202,60 @@ def _have(name: str, fallback: str) -> bool:
|
|
|
216
202
|
return _resolve_binary(name, fallback) is not None
|
|
217
203
|
|
|
218
204
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
result: str,
|
|
232
|
-
session_id: str,
|
|
233
|
-
tool_call_id: str,
|
|
234
|
-
) -> str | None:
|
|
235
|
-
tokenless_bin = _resolve_binary("tokenless", _TOKENLESS_FALLBACK)
|
|
236
|
-
if not tokenless_bin:
|
|
237
|
-
return None
|
|
238
|
-
|
|
239
|
-
parsed = _try_parse_json(result)
|
|
240
|
-
if not isinstance(parsed, (dict, list)):
|
|
241
|
-
return None
|
|
242
|
-
|
|
243
|
-
# 3-layer dispatch: shell tools use moderate truncation, API tools use zero-truncation
|
|
244
|
-
thresholds = get_thresholds(tool_name)
|
|
245
|
-
|
|
246
|
-
cmd = [
|
|
247
|
-
tokenless_bin, "compress-response",
|
|
248
|
-
"--agent-id", AGENT_ID,
|
|
249
|
-
"--truncate-strings-at", str(thresholds[0]),
|
|
250
|
-
"--truncate-arrays-at", str(thresholds[1]),
|
|
251
|
-
"--max-depth", str(thresholds[2]),
|
|
252
|
-
]
|
|
253
|
-
if session_id:
|
|
254
|
-
cmd.extend(["--session-id", session_id])
|
|
255
|
-
if tool_call_id:
|
|
256
|
-
cmd.extend(["--tool-use-id", tool_call_id])
|
|
257
|
-
|
|
258
|
-
proc = _run(cmd, result)
|
|
259
|
-
if not proc or proc.returncode != 0 or not proc.stdout.strip():
|
|
260
|
-
return None
|
|
261
|
-
|
|
262
|
-
compressed = proc.stdout.strip()
|
|
263
|
-
if len(compressed) >= len(result):
|
|
205
|
+
def _protocol_status(status: Any, result: str) -> str | None:
|
|
206
|
+
"""Map Hermes status, deriving it for hosts that omit the field."""
|
|
207
|
+
if isinstance(status, str) and status:
|
|
208
|
+
return {
|
|
209
|
+
"ok": "success",
|
|
210
|
+
"success": "success",
|
|
211
|
+
"error": "error",
|
|
212
|
+
"blocked": "denied",
|
|
213
|
+
"denied": "denied",
|
|
214
|
+
"interrupted": "interrupted",
|
|
215
|
+
}.get(status.lower())
|
|
216
|
+
if status not in (None, ""):
|
|
264
217
|
return None
|
|
265
|
-
return compressed
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
# ---------------------------------------------------------------------------
|
|
269
|
-
# 2. TOON Encoding (via tokenless compress-toon)
|
|
270
|
-
# ---------------------------------------------------------------------------
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
def _encode_toon(data: str, session_id: str = "", tool_call_id: str = "") -> tuple[str, int] | None:
|
|
274
|
-
tokenless_bin = _resolve_binary("tokenless", _TOKENLESS_FALLBACK)
|
|
275
|
-
if not tokenless_bin:
|
|
276
|
-
return None
|
|
277
|
-
|
|
278
|
-
parsed = _try_parse_json(data)
|
|
279
|
-
if not isinstance(parsed, (dict, list)):
|
|
280
|
-
return None
|
|
281
|
-
|
|
282
|
-
cmd = [tokenless_bin, "compress-toon", "--agent-id", AGENT_ID]
|
|
283
|
-
if session_id:
|
|
284
|
-
cmd.extend(["--session-id", session_id])
|
|
285
|
-
if tool_call_id:
|
|
286
|
-
cmd.extend(["--tool-use-id", tool_call_id])
|
|
287
|
-
|
|
288
|
-
proc = _run(cmd, data, timeout=1)
|
|
289
|
-
if not proc or proc.returncode != 0 or not proc.stdout.strip():
|
|
290
|
-
return None
|
|
291
|
-
|
|
292
|
-
toon_text = proc.stdout.strip()
|
|
293
|
-
if len(toon_text) >= len(data):
|
|
294
|
-
return None
|
|
295
|
-
|
|
296
|
-
savings_pct = 0
|
|
297
|
-
if len(data) > 0:
|
|
298
|
-
savings_pct = (len(data) - len(toon_text)) * 100 // len(data)
|
|
299
|
-
|
|
300
|
-
return toon_text, savings_pct
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
# ---------------------------------------------------------------------------
|
|
304
|
-
# 3. Tool Ready (via tokenless env-check)
|
|
305
|
-
# ---------------------------------------------------------------------------
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
def _env_check(tool_name: str) -> str | None:
|
|
309
|
-
"""Run tool-ready env-check and return feedback if tool is not ready."""
|
|
310
|
-
tokenless_bin = _resolve_binary("tokenless", _TOKENLESS_FALLBACK)
|
|
311
|
-
if not tokenless_bin:
|
|
312
|
-
return None
|
|
313
|
-
|
|
314
|
-
proc = _run([tokenless_bin, "env-check", "--tool", tool_name, "--json"], "", timeout=5)
|
|
315
|
-
if not proc or not proc.stdout.strip():
|
|
316
|
-
return None
|
|
317
|
-
|
|
318
218
|
try:
|
|
319
|
-
parsed = json.loads(
|
|
219
|
+
parsed = json.loads(result)
|
|
320
220
|
except json.JSONDecodeError:
|
|
321
|
-
return
|
|
221
|
+
return "success"
|
|
222
|
+
return "error" if isinstance(parsed, dict) and parsed.get("error") else "success"
|
|
322
223
|
|
|
323
|
-
status = parsed.get("status", "UNKNOWN")
|
|
324
|
-
if status in ("UNKNOWN", "READY"):
|
|
325
|
-
return None
|
|
326
|
-
|
|
327
|
-
# Attempt auto-fix
|
|
328
|
-
proc = _run([tokenless_bin, "env-check", "--tool", tool_name, "--fix", "--json"], "", timeout=10)
|
|
329
|
-
if not proc or not proc.stdout.strip():
|
|
330
|
-
return _not_ready_msg(tool_name)
|
|
331
|
-
|
|
332
|
-
try:
|
|
333
|
-
fix_parsed = json.loads(proc.stdout)
|
|
334
|
-
except json.JSONDecodeError:
|
|
335
|
-
return _not_ready_msg(tool_name)
|
|
336
224
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
def _not_ready_msg(tool_name: str) -> str:
|
|
345
|
-
return f"[tokenless:ready] {tool_name}: NOT_READY. Skip retry."
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
# ---------------------------------------------------------------------------
|
|
349
|
-
# 4. Command Rewriting (via rtk rewrite)
|
|
350
|
-
# ---------------------------------------------------------------------------
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
def _try_rewrite(
|
|
354
|
-
args: Any,
|
|
355
|
-
session_id: str,
|
|
356
|
-
tool_call_id: str,
|
|
357
|
-
) -> dict[str, str] | None:
|
|
358
|
-
"""Attempt RTK command rewrite for terminal tool calls.
|
|
225
|
+
def _content_origin(tool_name: str) -> str:
|
|
226
|
+
if tool_name in _SKIP_TOOLS:
|
|
227
|
+
return "file_content"
|
|
228
|
+
if tool_name in _SHELL_TOOLS:
|
|
229
|
+
return "command_output"
|
|
230
|
+
return "api_response"
|
|
359
231
|
|
|
360
|
-
Calls ``rtk rewrite <command>`` — a pure text substitution that never
|
|
361
|
-
executes the command. On success, returns a block directive suggesting
|
|
362
|
-
the rewritten command so the agent re-executes with the optimized version.
|
|
363
|
-
"""
|
|
364
|
-
rtk_bin = _resolve_binary("rtk", _RTK_FALLBACK)
|
|
365
|
-
if not rtk_bin:
|
|
366
|
-
return None
|
|
367
232
|
|
|
233
|
+
def _output_optimization(args: Any) -> str:
|
|
234
|
+
"""Recognize the attributed RTK wrapper in the command Hermes executed."""
|
|
368
235
|
if not isinstance(args, dict):
|
|
369
|
-
return
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
return None
|
|
374
|
-
|
|
375
|
-
# Version guard — non-fatal
|
|
236
|
+
return "none"
|
|
237
|
+
command = args.get("command")
|
|
238
|
+
if not isinstance(command, str):
|
|
239
|
+
return "none"
|
|
376
240
|
try:
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
)
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
if tool_call_id:
|
|
396
|
-
env["TOKENLESS_TOOL_USE_ID"] = tool_call_id
|
|
397
|
-
|
|
398
|
-
proc = subprocess.run(
|
|
399
|
-
[rtk_bin, "rewrite", command],
|
|
400
|
-
capture_output=True, text=True, timeout=5, env=env,
|
|
401
|
-
)
|
|
402
|
-
|
|
403
|
-
# Exit code protocol (from rtk rewrite_cmd.rs):
|
|
404
|
-
# 0 = rewrite available, Allow verdict (auto-allow by permission rule)
|
|
405
|
-
# 1 = no RTK equivalent (passthrough)
|
|
406
|
-
# 2 = deny rule matched (let Hermes handle)
|
|
407
|
-
# 3 = Ask/Default verdict (rewrite available but permission model requires
|
|
408
|
-
# user confirmation; in non-interactive hook context, treat as valid
|
|
409
|
-
# rewrite since the intent is token optimization, not permission gating)
|
|
410
|
-
if proc.returncode == 1 or proc.returncode == 2:
|
|
411
|
-
return None
|
|
412
|
-
if proc.returncode != 0 and proc.returncode != 3:
|
|
413
|
-
return None
|
|
414
|
-
|
|
415
|
-
rewritten = proc.stdout.strip()
|
|
416
|
-
if not rewritten or rewritten == command:
|
|
417
|
-
return None
|
|
418
|
-
|
|
419
|
-
logger.info("tokenless: rtk rewrite %s → %s", command, rewritten)
|
|
420
|
-
return {
|
|
421
|
-
"action": "block",
|
|
422
|
-
"message": f"[tokenless:rewrite] Re-execute as: {rewritten}",
|
|
423
|
-
}
|
|
241
|
+
lexer = shlex.shlex(command, posix=True, punctuation_chars=True)
|
|
242
|
+
lexer.whitespace_split = True
|
|
243
|
+
tokens = list(lexer)
|
|
244
|
+
except ValueError:
|
|
245
|
+
return "none"
|
|
246
|
+
for index, token in enumerate(tokens):
|
|
247
|
+
wrapper = tokens[index : index + 6]
|
|
248
|
+
if len(wrapper) < 6 or token != "env":
|
|
249
|
+
continue
|
|
250
|
+
if (
|
|
251
|
+
wrapper[1] == f"TOKENLESS_AGENT_ID={AGENT_ID}"
|
|
252
|
+
and wrapper[2].startswith("TOKENLESS_SESSION_ID=")
|
|
253
|
+
and wrapper[3].startswith("TOKENLESS_TOOL_USE_ID=")
|
|
254
|
+
and wrapper[4].startswith("TOKENLESS_DATA_DIR=")
|
|
255
|
+
and os.path.basename(wrapper[5]) == "rtk"
|
|
256
|
+
):
|
|
257
|
+
return "rtk"
|
|
258
|
+
return "none"
|
|
424
259
|
|
|
425
260
|
|
|
426
261
|
# ---------------------------------------------------------------------------
|
|
@@ -444,28 +279,39 @@ def on_pre_tool_call(
|
|
|
444
279
|
tool_call_id: str = "",
|
|
445
280
|
**kwargs: Any,
|
|
446
281
|
) -> dict[str, str] | None:
|
|
447
|
-
"""
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
282
|
+
"""Ask Core for an RTK rewrite and translate it to a Hermes block."""
|
|
283
|
+
if tool_name not in _SHELL_TOOLS or not isinstance(args, dict):
|
|
284
|
+
return None
|
|
285
|
+
tokenless_bin = _resolve_binary("tokenless", _TOKENLESS_FALLBACK)
|
|
286
|
+
if not tokenless_bin:
|
|
287
|
+
return None
|
|
288
|
+
request = build_pre_tool_request(
|
|
289
|
+
args,
|
|
290
|
+
AGENT_ID,
|
|
291
|
+
tool_name,
|
|
292
|
+
"command",
|
|
293
|
+
str(session_id),
|
|
294
|
+
str(tool_call_id),
|
|
295
|
+
replace_arguments=False,
|
|
296
|
+
block_and_suggest=True,
|
|
297
|
+
)
|
|
298
|
+
result = run_compress(tokenless_bin, request, _COMPRESS_TIMEOUT_SECONDS, "pre_tool")
|
|
299
|
+
if not isinstance(result, dict):
|
|
300
|
+
return None
|
|
301
|
+
rewritten_args = result.get("arguments")
|
|
302
|
+
rewritten = rewritten_args.get("command") if isinstance(rewritten_args, dict) else None
|
|
303
|
+
if (
|
|
304
|
+
result.get("action") != "block_and_suggest"
|
|
305
|
+
or result.get("output_optimization") != "rtk"
|
|
306
|
+
or not isinstance(rewritten, str)
|
|
307
|
+
or rewritten == args.get("command")
|
|
308
|
+
):
|
|
309
|
+
return None
|
|
310
|
+
logger.info("tokenless: Core rewrote %s", tool_name)
|
|
311
|
+
return {
|
|
312
|
+
"action": "block",
|
|
313
|
+
"message": f"[tokenless:rewrite] Re-execute as: {rewritten}",
|
|
314
|
+
}
|
|
469
315
|
|
|
470
316
|
|
|
471
317
|
def on_transform_tool_result(
|
|
@@ -476,83 +322,81 @@ def on_transform_tool_result(
|
|
|
476
322
|
session_id: str = "",
|
|
477
323
|
tool_call_id: str = "",
|
|
478
324
|
duration_ms: int = 0,
|
|
325
|
+
status: str = "",
|
|
479
326
|
**kwargs: Any,
|
|
480
327
|
) -> str | None:
|
|
481
|
-
"""
|
|
482
|
-
|
|
483
|
-
Replaces the tool result string with a compressed/TOON-encoded version.
|
|
484
|
-
Runs after post_tool_call; first valid string return wins.
|
|
485
|
-
|
|
486
|
-
Content retrieval tools (Read/Glob/Grep) are skipped entirely.
|
|
487
|
-
Shell/exec tools (Bash/Shell) use moderate truncation (64K/128/8).
|
|
488
|
-
All other tools use zero-truncation compress-response + TOON.
|
|
489
|
-
"""
|
|
490
|
-
if not _have("tokenless", _TOKENLESS_FALLBACK):
|
|
328
|
+
"""Send one final Hermes result to Core and apply its disposition."""
|
|
329
|
+
if not isinstance(result, str):
|
|
491
330
|
return None
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
if tool_name in _SKIP_TOOLS:
|
|
331
|
+
protocol_status = _protocol_status(status, result)
|
|
332
|
+
if protocol_status is None:
|
|
495
333
|
return None
|
|
496
|
-
|
|
497
|
-
if not
|
|
498
|
-
return None
|
|
499
|
-
|
|
500
|
-
# Skip skill files (YAML frontmatter)
|
|
501
|
-
if _is_skill_file(result):
|
|
502
|
-
return None
|
|
503
|
-
|
|
504
|
-
# Skip small responses
|
|
505
|
-
if len(result) < _MIN_RESPONSE_LEN:
|
|
334
|
+
tokenless_bin = _resolve_binary("tokenless", _TOKENLESS_FALLBACK)
|
|
335
|
+
if not tokenless_bin:
|
|
506
336
|
return None
|
|
337
|
+
output_optimization = _output_optimization(args)
|
|
338
|
+
retrieve_result = protocol_status == "success" and is_tokenless_retrieve_command(
|
|
339
|
+
tool_name, args
|
|
340
|
+
)
|
|
507
341
|
|
|
508
|
-
#
|
|
509
|
-
|
|
510
|
-
|
|
342
|
+
# Hermes's terminal tool returns a JSON envelope whose `output` field is
|
|
343
|
+
# the model-visible command output. Compress that field so structured JSON
|
|
344
|
+
# produced by the command remains visible to JsonCompressor, then restore
|
|
345
|
+
# the host envelope below. Other tools already expose their model-bound
|
|
346
|
+
# result directly and must keep the existing path.
|
|
347
|
+
shell_envelope = None
|
|
348
|
+
content = result
|
|
349
|
+
if tool_name in _SHELL_TOOLS:
|
|
350
|
+
try:
|
|
351
|
+
parsed_result = json.loads(result)
|
|
352
|
+
except json.JSONDecodeError:
|
|
353
|
+
parsed_result = None
|
|
354
|
+
if isinstance(parsed_result, dict) and isinstance(parsed_result.get("output"), str):
|
|
355
|
+
shell_envelope = parsed_result
|
|
356
|
+
content = parsed_result["output"]
|
|
357
|
+
|
|
358
|
+
request = build_post_tool_request(
|
|
359
|
+
content,
|
|
360
|
+
AGENT_ID,
|
|
361
|
+
tool_name,
|
|
362
|
+
protocol_status,
|
|
363
|
+
_content_origin(tool_name),
|
|
364
|
+
output_optimization,
|
|
365
|
+
result_kind="retrieve" if retrieve_result else "tool",
|
|
366
|
+
recovery={
|
|
367
|
+
"kind": (
|
|
368
|
+
"shell"
|
|
369
|
+
if (
|
|
370
|
+
protocol_status == "success"
|
|
371
|
+
and output_optimization == "none"
|
|
372
|
+
and not retrieve_result
|
|
373
|
+
and tokenless_retrieve_command_available()
|
|
374
|
+
)
|
|
375
|
+
else "none"
|
|
376
|
+
)
|
|
377
|
+
},
|
|
378
|
+
session_id=str(session_id),
|
|
379
|
+
tool_use_id=str(tool_call_id),
|
|
380
|
+
replace_output=True,
|
|
381
|
+
replace_with_text=True,
|
|
382
|
+
)
|
|
383
|
+
response = run_compress(tokenless_bin, request, _COMPRESS_TIMEOUT_SECONDS, "post_tool")
|
|
384
|
+
if not isinstance(response, dict):
|
|
511
385
|
return None
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
current = compressed if compressed else result
|
|
521
|
-
|
|
522
|
-
# Step 2: TOON encoding — only for payloads at or above the minimum
|
|
523
|
-
# threshold; small JSON gains near-zero chars from TOON but would still
|
|
524
|
-
# pay the full encode cost on every tool result.
|
|
525
|
-
toon_result = None
|
|
526
|
-
if len(current) >= _MIN_TOON_CHARS:
|
|
527
|
-
toon_result = _encode_toon(current, str(session_id), str(tool_call_id))
|
|
528
|
-
used_compression = compressed is not None
|
|
529
|
-
used_toon = toon_result is not None
|
|
530
|
-
|
|
531
|
-
if not used_compression and not used_toon:
|
|
386
|
+
if response.get("disposition") == "applied":
|
|
387
|
+
output = response.get("output")
|
|
388
|
+
if isinstance(output, str):
|
|
389
|
+
logger.info("tokenless: Core optimized %s", tool_name)
|
|
390
|
+
if shell_envelope is not None:
|
|
391
|
+
shell_envelope["output"] = output
|
|
392
|
+
return json.dumps(shell_envelope, ensure_ascii=False)
|
|
393
|
+
return output
|
|
532
394
|
return None
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
final_len = len(toon_text)
|
|
539
|
-
savings_label = (
|
|
540
|
-
"response compressed + TOON encoded"
|
|
541
|
-
if used_compression
|
|
542
|
-
else "TOON encoded"
|
|
543
|
-
)
|
|
544
|
-
else:
|
|
545
|
-
final = current # type: ignore[assignment]
|
|
546
|
-
final_len = len(final)
|
|
547
|
-
savings_pct = (original_len - final_len) * 100 // original_len if original_len else 0
|
|
548
|
-
savings_label = "response compressed"
|
|
549
|
-
|
|
550
|
-
logger.info(
|
|
551
|
-
"tokenless: %s %s: %d -> %d chars (%d%% reduction)",
|
|
552
|
-
savings_label, tool_name, original_len, final_len, savings_pct,
|
|
553
|
-
)
|
|
554
|
-
|
|
555
|
-
return final
|
|
395
|
+
if response.get("disposition") == "tool_error":
|
|
396
|
+
additional_context = response.get("additional_context")
|
|
397
|
+
if isinstance(additional_context, str) and additional_context:
|
|
398
|
+
return f"{result}\n\n{additional_context}"
|
|
399
|
+
return None
|
|
556
400
|
|
|
557
401
|
|
|
558
402
|
# ---------------------------------------------------------------------------
|
|
@@ -567,16 +411,8 @@ def register(ctx: Any) -> None:
|
|
|
567
411
|
ctx.register_hook("pre_tool_call", on_pre_tool_call)
|
|
568
412
|
ctx.register_hook("transform_tool_result", on_transform_tool_result)
|
|
569
413
|
|
|
570
|
-
|
|
571
|
-
features: list[str] = []
|
|
572
|
-
if _have("tokenless", _TOKENLESS_FALLBACK):
|
|
573
|
-
features.append("response-compression")
|
|
574
|
-
features.append("toon-encoding")
|
|
575
|
-
features.append("tool-ready")
|
|
576
|
-
if _have("rtk", _RTK_FALLBACK):
|
|
577
|
-
features.append("rtk-rewrite")
|
|
578
|
-
|
|
414
|
+
features = ["pre-tool", "post-tool"] if _have("tokenless", _TOKENLESS_FALLBACK) else []
|
|
579
415
|
logger.info(
|
|
580
416
|
"tokenless: Hermes plugin registered — active features: %s",
|
|
581
|
-
", ".join(features) if features else "none (install tokenless
|
|
417
|
+
", ".join(features) if features else "none (install tokenless binary)",
|
|
582
418
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
name: tokenless
|
|
2
|
-
version: "0.
|
|
3
|
-
description: "
|
|
2
|
+
version: "0.8.0"
|
|
3
|
+
description: "Tokenless lifecycle integration for Hermes Agent — Core-owned command rewriting and result optimization"
|
|
4
4
|
author: ANOLISA
|
|
5
5
|
requires_env: []
|
|
6
6
|
provides_hooks:
|