@softspark/ai-toolkit 4.16.1 → 4.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +84 -0
- package/README.md +13 -19
- package/app/.claude-plugin/plugin.json +1 -1
- package/app/hooks/session-end.sh +1 -13
- package/app/hooks.json +0 -10
- package/app/plugins/README.md +16 -4
- package/app/plugins/rtk-pack/README.md +123 -0
- package/app/plugins/rtk-pack/hooks/rewrite.sh +79 -0
- package/app/plugins/rtk-pack/plugin.json +60 -0
- package/app/plugins/rtk-pack/scripts/init.py +252 -0
- package/app/plugins/rtk-pack/scripts/status.py +105 -0
- package/benchmarks/ecosystem-doctor-snapshot.json +14 -15
- package/bin/ai-toolkit.js +15 -2
- package/kb/history/completed/output-filter-retirement-20260726.md +128 -0
- package/kb/history/completed/rtk-pack-integration-20260726.md +710 -0
- package/kb/procedures/maintenance-sop.md +1 -1
- package/kb/procedures/release-preparation-sop.md +8 -3
- package/kb/procedures/rtk-upstream-sync-sop.md +279 -0
- package/kb/reference/architecture-overview.md +2 -3
- package/kb/reference/cli-reference.md +4 -14
- package/kb/reference/enterprise-config-guide.md +1 -21
- package/kb/reference/hooks-catalog.md +3 -60
- package/kb/reference/plugin-pack-conventions.md +17 -2
- package/kb/reference/supported-tools-registry.md +0 -4
- package/llms-full.txt +1171 -402
- package/llms.txt +3 -1
- package/manifest.json +147 -36
- package/package.json +1 -2
- package/scripts/audit_skills.py +21 -0
- package/scripts/claude_app.py +2 -21
- package/scripts/config_cli.py +4 -0
- package/scripts/config_merger.py +0 -17
- package/scripts/config_validator.py +11 -138
- package/scripts/doctor.py +3 -20
- package/scripts/install.py +2 -1
- package/scripts/install_steps/ai_tools.py +28 -99
- package/scripts/install_steps/hooks.py +26 -24
- package/scripts/merge-hooks.py +33 -2
- package/scripts/output_filter_retirement.py +395 -0
- package/scripts/plugin.py +136 -16
- package/scripts/schemas/ai-toolkit-config.schema.json +0 -60
- package/scripts/uninstall.py +13 -27
- package/scripts/verify_rtk_binary.py +335 -0
- package/app/hooks/filter-tool-output.sh +0 -76
- package/app/output-filter-policy.json +0 -15
- package/benchmarks/output-filter/README.md +0 -11
- package/benchmarks/output-filter/scenarios.json +0 -25
- package/kb/reference/tool-output-filter.md +0 -288
- package/scripts/benchmark_output_filter.py +0 -343
- package/scripts/output_filter_cli.py +0 -347
- package/scripts/output_filter_hook.py +0 -23
- package/scripts/tool_output_filter/__init__.py +0 -33
- package/scripts/tool_output_filter/contracts.py +0 -173
- package/scripts/tool_output_filter/engine.py +0 -260
- package/scripts/tool_output_filter/hook_runtime.py +0 -369
- package/scripts/tool_output_filter/input.py +0 -56
- package/scripts/tool_output_filter/invariants.py +0 -40
- package/scripts/tool_output_filter/policy.py +0 -153
- package/scripts/tool_output_filter/profiles/__init__.py +0 -68
- package/scripts/tool_output_filter/profiles/repeat_lines.py +0 -71
- package/scripts/tool_output_filter/profiles/tap_success.py +0 -154
- package/scripts/tool_output_filter/recovery.py +0 -846
- package/scripts/tool_output_filter/telemetry.py +0 -13
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fetch, verify and install the rtk binary for this platform.
|
|
3
|
+
|
|
4
|
+
Run by `ai-toolkit plugin install rtk-pack` (scripts/plugin.py invokes
|
|
5
|
+
`init.py` if present). This is the only point at which the pack touches the
|
|
6
|
+
network; nothing is fetched at runtime.
|
|
7
|
+
|
|
8
|
+
Design constraints from kb/history/completed/rtk-pack-integration-20260726.md section 6:
|
|
9
|
+
|
|
10
|
+
- Verify before install. A digest mismatch aborts and removes the partial
|
|
11
|
+
download; a half-installed binary is worse than none.
|
|
12
|
+
- A fetch failure is not an install failure. The pack degrades to inert and
|
|
13
|
+
says so, matching how the core behaves when jq is missing. plugin.py only
|
|
14
|
+
warns on a non-zero exit, so the message has to carry the meaning.
|
|
15
|
+
- Never silent. Every path that ends without a working binary prints why.
|
|
16
|
+
|
|
17
|
+
Stdlib only.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import platform
|
|
25
|
+
import shutil
|
|
26
|
+
import ssl
|
|
27
|
+
import sys
|
|
28
|
+
import tarfile
|
|
29
|
+
import tempfile
|
|
30
|
+
import urllib.error
|
|
31
|
+
import urllib.request
|
|
32
|
+
import zipfile
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
PACK_NAME = "rtk-pack"
|
|
36
|
+
TOOLKIT_DATA_DIR = Path(os.environ.get("AI_TOOLKIT_DATA_DIR", Path.home() / ".softspark" / "ai-toolkit"))
|
|
37
|
+
INSTALL_DIR = TOOLKIT_DATA_DIR / "plugin-scripts" / PACK_NAME / "bin"
|
|
38
|
+
VERSION_FILE = TOOLKIT_DATA_DIR / "plugin-scripts" / PACK_NAME / "version.json"
|
|
39
|
+
|
|
40
|
+
DOWNLOAD_TIMEOUT = 120
|
|
41
|
+
CHUNK = 1 << 16
|
|
42
|
+
|
|
43
|
+
# Bounded so a wrong URL cannot fill the disk. The largest real asset is ~4 MB.
|
|
44
|
+
MAX_ASSET_BYTES = 64 * 1024 * 1024
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class InstallError(Exception):
|
|
48
|
+
"""Anything that leaves the pack without a usable binary."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def manifest_path() -> Path:
|
|
52
|
+
"""The pack manifest, resolved relative to this script's source location."""
|
|
53
|
+
return Path(__file__).resolve().parent.parent / "plugin.json"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def detect_platform() -> str:
|
|
57
|
+
"""Map this host to an asset key in plugin.json.
|
|
58
|
+
|
|
59
|
+
Linux x86_64 is served the static musl build, which runs on glibc; upstream
|
|
60
|
+
routes Linux x86_64 to musl in its own Homebrew formula for the same reason.
|
|
61
|
+
"""
|
|
62
|
+
system = platform.system().lower()
|
|
63
|
+
machine = platform.machine().lower()
|
|
64
|
+
arch = {
|
|
65
|
+
"x86_64": "x86_64",
|
|
66
|
+
"amd64": "x86_64",
|
|
67
|
+
"arm64": "arm64" if system == "darwin" else "aarch64",
|
|
68
|
+
"aarch64": "arm64" if system == "darwin" else "aarch64",
|
|
69
|
+
}.get(machine)
|
|
70
|
+
if arch is None:
|
|
71
|
+
raise InstallError(f"unsupported architecture: {platform.machine()}")
|
|
72
|
+
if system not in ("darwin", "linux", "windows"):
|
|
73
|
+
raise InstallError(f"unsupported platform: {platform.system()}")
|
|
74
|
+
return f"{system}-{arch}"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def asset_url(binary: dict, asset: dict) -> str:
|
|
78
|
+
"""Where to fetch this asset from.
|
|
79
|
+
|
|
80
|
+
RTK_PACK_RELEASE_BASE_URL points the fetch at a mirror instead of GitHub,
|
|
81
|
+
for networks that cannot reach it directly. The digest check is unchanged,
|
|
82
|
+
so a mirror serving different bytes is rejected exactly like a corrupt
|
|
83
|
+
download.
|
|
84
|
+
"""
|
|
85
|
+
base = os.environ.get("RTK_PACK_RELEASE_BASE_URL", "").rstrip("/")
|
|
86
|
+
if base:
|
|
87
|
+
return f"{base}/{asset['file']}"
|
|
88
|
+
return (
|
|
89
|
+
f"https://github.com/{binary['release_repo']}/releases/download/"
|
|
90
|
+
f"{binary['release_tag']}/{asset['file']}"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def download(url: str, dest: Path) -> str:
|
|
95
|
+
"""Fetch to dest, returning the SHA-256 of what actually landed."""
|
|
96
|
+
digest = hashlib.sha256()
|
|
97
|
+
total = 0
|
|
98
|
+
request = urllib.request.Request(url, headers={"User-Agent": "ai-toolkit-rtk-pack"})
|
|
99
|
+
# A default TLS context for https; file:// mirrors (and the tests) take the
|
|
100
|
+
# handler that ignores it.
|
|
101
|
+
kwargs = {"timeout": DOWNLOAD_TIMEOUT}
|
|
102
|
+
if url.startswith("https:"):
|
|
103
|
+
kwargs["context"] = ssl.create_default_context()
|
|
104
|
+
try:
|
|
105
|
+
with urllib.request.urlopen(request, **kwargs) as response:
|
|
106
|
+
with dest.open("wb") as handle:
|
|
107
|
+
while True:
|
|
108
|
+
chunk = response.read(CHUNK)
|
|
109
|
+
if not chunk:
|
|
110
|
+
break
|
|
111
|
+
total += len(chunk)
|
|
112
|
+
if total > MAX_ASSET_BYTES:
|
|
113
|
+
raise InstallError(f"asset exceeds {MAX_ASSET_BYTES} bytes, refusing to continue")
|
|
114
|
+
digest.update(chunk)
|
|
115
|
+
handle.write(chunk)
|
|
116
|
+
except urllib.error.HTTPError as exc:
|
|
117
|
+
raise InstallError(f"HTTP {exc.code} fetching {url}") from exc
|
|
118
|
+
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
119
|
+
raise InstallError(f"could not fetch {url}: {exc}") from exc
|
|
120
|
+
if total == 0:
|
|
121
|
+
raise InstallError(f"empty response from {url}")
|
|
122
|
+
return digest.hexdigest()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def extract_member(archive: Path, member: str, dest: Path) -> None:
|
|
126
|
+
"""Pull exactly one flat entry out of the archive.
|
|
127
|
+
|
|
128
|
+
The archives are built to hold a single flat binary and CI asserts it, so
|
|
129
|
+
anything else means the asset is not what the manifest claims and is
|
|
130
|
+
refused rather than extracted.
|
|
131
|
+
"""
|
|
132
|
+
if archive.suffix == ".zip":
|
|
133
|
+
with zipfile.ZipFile(archive) as zf:
|
|
134
|
+
names = zf.namelist()
|
|
135
|
+
if names != [member]:
|
|
136
|
+
raise InstallError(f"expected exactly [{member}] in {archive.name}, found {names}")
|
|
137
|
+
with zf.open(member) as src, dest.open("wb") as out:
|
|
138
|
+
shutil.copyfileobj(src, out)
|
|
139
|
+
return
|
|
140
|
+
|
|
141
|
+
with tarfile.open(archive, "r:gz") as tf:
|
|
142
|
+
names = tf.getnames()
|
|
143
|
+
if names != [member]:
|
|
144
|
+
raise InstallError(f"expected exactly [{member}] in {archive.name}, found {names}")
|
|
145
|
+
extracted = tf.extractfile(member)
|
|
146
|
+
if extracted is None:
|
|
147
|
+
raise InstallError(f"{member} in {archive.name} is not a regular file")
|
|
148
|
+
with extracted as src, dest.open("wb") as out:
|
|
149
|
+
shutil.copyfileobj(src, out)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def already_installed(asset: dict, target: Path) -> bool:
|
|
153
|
+
"""True when this exact asset is already in place.
|
|
154
|
+
|
|
155
|
+
`ai-toolkit plugin install --editor all` calls the init step twice in a
|
|
156
|
+
single command (plugin.py invokes _copy_plugin_scripts per editor), so
|
|
157
|
+
without this the binary is downloaded twice. It also makes a re-install
|
|
158
|
+
cheap instead of re-fetching.
|
|
159
|
+
"""
|
|
160
|
+
if not target.is_file():
|
|
161
|
+
return False
|
|
162
|
+
try:
|
|
163
|
+
record = json.loads(VERSION_FILE.read_text(encoding="utf-8"))
|
|
164
|
+
except (OSError, json.JSONDecodeError):
|
|
165
|
+
return False
|
|
166
|
+
return record.get("sha256") == asset["sha256"] and record.get("asset") == asset["file"]
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def install(manifest: dict) -> dict | None:
|
|
170
|
+
binary = manifest.get("binary")
|
|
171
|
+
if not binary:
|
|
172
|
+
raise InstallError("plugin.json has no 'binary' section")
|
|
173
|
+
|
|
174
|
+
key = detect_platform()
|
|
175
|
+
asset = binary.get("assets", {}).get(key)
|
|
176
|
+
if asset is None:
|
|
177
|
+
raise InstallError(f"no asset published for {key}")
|
|
178
|
+
|
|
179
|
+
url = asset_url(binary, asset)
|
|
180
|
+
install_name = binary.get("install_name", "rtk")
|
|
181
|
+
if platform.system().lower() == "windows":
|
|
182
|
+
install_name += ".exe"
|
|
183
|
+
target = INSTALL_DIR / install_name
|
|
184
|
+
|
|
185
|
+
if already_installed(asset, target):
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
189
|
+
staged = Path(tmp) / asset["file"]
|
|
190
|
+
actual = download(url, staged)
|
|
191
|
+
if actual != asset["sha256"]:
|
|
192
|
+
# Nothing is installed on a mismatch. The temp dir takes the
|
|
193
|
+
# partial download with it.
|
|
194
|
+
raise InstallError(
|
|
195
|
+
f"digest mismatch for {asset['file']}: "
|
|
196
|
+
f"expected {asset['sha256']}, got {actual}"
|
|
197
|
+
)
|
|
198
|
+
unpacked = Path(tmp) / install_name
|
|
199
|
+
extract_member(staged, asset["member"], unpacked)
|
|
200
|
+
|
|
201
|
+
INSTALL_DIR.mkdir(parents=True, exist_ok=True)
|
|
202
|
+
shutil.move(str(unpacked), str(target))
|
|
203
|
+
|
|
204
|
+
target.chmod(target.stat().st_mode | 0o111)
|
|
205
|
+
|
|
206
|
+
record = {
|
|
207
|
+
"pack_version": manifest.get("version"),
|
|
208
|
+
"upstream_version": manifest.get("upstream", {}).get("version"),
|
|
209
|
+
"release_tag": binary["release_tag"],
|
|
210
|
+
"platform": key,
|
|
211
|
+
"asset": asset["file"],
|
|
212
|
+
"sha256": asset["sha256"],
|
|
213
|
+
"binary": str(target),
|
|
214
|
+
}
|
|
215
|
+
VERSION_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
216
|
+
VERSION_FILE.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8")
|
|
217
|
+
return record
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def main() -> int:
|
|
221
|
+
try:
|
|
222
|
+
manifest = json.loads(manifest_path().read_text(encoding="utf-8"))
|
|
223
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
224
|
+
print(f"rtk-pack: cannot read plugin.json: {exc}", file=sys.stderr)
|
|
225
|
+
return 1
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
record = install(manifest)
|
|
229
|
+
if record is None:
|
|
230
|
+
print("rtk already installed and digest matches, nothing to do")
|
|
231
|
+
return 0
|
|
232
|
+
except InstallError as exc:
|
|
233
|
+
# plugin.py prints this as "WARN init failed" and continues, which is
|
|
234
|
+
# the intended behaviour: the hook stays wired but finds no binary and
|
|
235
|
+
# passes every command through untouched.
|
|
236
|
+
print(
|
|
237
|
+
f"rtk-pack: no binary installed ({exc}). "
|
|
238
|
+
"The pack is inert: commands are passed through unchanged. "
|
|
239
|
+
"Re-run 'ai-toolkit plugin install rtk-pack' once the cause is resolved.",
|
|
240
|
+
file=sys.stderr,
|
|
241
|
+
)
|
|
242
|
+
return 1
|
|
243
|
+
|
|
244
|
+
print(
|
|
245
|
+
f"rtk {record['upstream_version']} installed for {record['platform']} "
|
|
246
|
+
f"({record['binary']}), digest verified"
|
|
247
|
+
)
|
|
248
|
+
return 0
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
if __name__ == "__main__":
|
|
252
|
+
sys.exit(main())
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Report rtk-pack health: binary, digest, and hook wiring.
|
|
3
|
+
|
|
4
|
+
Invoked by `ai-toolkit plugin status` and `ai-toolkit doctor` through the
|
|
5
|
+
generic scripts/status.py hook in scripts/plugin.py.
|
|
6
|
+
|
|
7
|
+
The point of this script is to distinguish "installed" from "working". The pack
|
|
8
|
+
can be recorded as installed, have its hook wired, and still do nothing at all
|
|
9
|
+
because the binary never downloaded. That is the failure mode this project has
|
|
10
|
+
already shipped once, so it gets its own line rather than being inferred.
|
|
11
|
+
|
|
12
|
+
Stdlib only. Never raises; exit code is always 0.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
PACK_NAME = "rtk-pack"
|
|
24
|
+
TOOLKIT_DATA_DIR = Path(
|
|
25
|
+
os.environ.get("AI_TOOLKIT_DATA_DIR", Path.home() / ".softspark" / "ai-toolkit")
|
|
26
|
+
)
|
|
27
|
+
PACK_STATE_DIR = TOOLKIT_DATA_DIR / "plugin-scripts" / PACK_NAME
|
|
28
|
+
VERSION_FILE = PACK_STATE_DIR / "version.json"
|
|
29
|
+
HOOK_FILE = TOOLKIT_DATA_DIR / "hooks" / f"plugin-{PACK_NAME}-rewrite.sh"
|
|
30
|
+
CLAUDE_SETTINGS = Path.home() / ".claude" / "settings.json"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def find_binary() -> Path | None:
|
|
34
|
+
for candidate in (PACK_STATE_DIR / "bin" / "rtk", PACK_STATE_DIR / "bin" / "rtk.exe"):
|
|
35
|
+
if candidate.is_file():
|
|
36
|
+
return candidate
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def digest_of(path: Path) -> str:
|
|
41
|
+
h = hashlib.sha256()
|
|
42
|
+
with path.open("rb") as handle:
|
|
43
|
+
for chunk in iter(lambda: handle.read(1 << 16), b""):
|
|
44
|
+
h.update(chunk)
|
|
45
|
+
return h.hexdigest()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def hook_registered() -> bool:
|
|
49
|
+
"""True when settings.json still points at this pack's hook."""
|
|
50
|
+
try:
|
|
51
|
+
settings = json.loads(CLAUDE_SETTINGS.read_text(encoding="utf-8"))
|
|
52
|
+
except (OSError, json.JSONDecodeError):
|
|
53
|
+
return False
|
|
54
|
+
return f"plugin-{PACK_NAME}-rewrite.sh" in json.dumps(settings.get("hooks", {}))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def main() -> int:
|
|
58
|
+
binary = find_binary()
|
|
59
|
+
if binary is None:
|
|
60
|
+
print("rtk binary: MISSING — the hook is inert and every command runs unchanged")
|
|
61
|
+
print(" fix: ai-toolkit plugin install rtk-pack")
|
|
62
|
+
return 0
|
|
63
|
+
|
|
64
|
+
record: dict = {}
|
|
65
|
+
try:
|
|
66
|
+
record = json.loads(VERSION_FILE.read_text(encoding="utf-8"))
|
|
67
|
+
except (OSError, json.JSONDecodeError):
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
upstream = record.get("upstream_version", "unknown")
|
|
71
|
+
print(f"rtk binary: {binary} (upstream {upstream})")
|
|
72
|
+
|
|
73
|
+
# The recorded digest is of the archive, not the extracted binary, so it
|
|
74
|
+
# cannot be recomputed here. Report the binary's own digest and whether the
|
|
75
|
+
# install record still exists, and say which is which rather than implying
|
|
76
|
+
# a verification that is not happening.
|
|
77
|
+
if record:
|
|
78
|
+
print(f" install record: {record.get('asset', '?')} sha256 {record.get('sha256', '?')[:16]}…")
|
|
79
|
+
else:
|
|
80
|
+
print(" install record: MISSING — reinstall to restore provenance")
|
|
81
|
+
print(f" binary sha256: {digest_of(binary)[:16]}…")
|
|
82
|
+
|
|
83
|
+
if not os.access(binary, os.X_OK):
|
|
84
|
+
print(" executable: NO — the hook cannot run it")
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
proc = subprocess.run([str(binary), "--version"], capture_output=True, text=True, timeout=15)
|
|
88
|
+
reported = (proc.stdout.strip() or proc.stderr.strip()).splitlines()[0] if proc.returncode == 0 else ""
|
|
89
|
+
except (OSError, subprocess.SubprocessError, IndexError):
|
|
90
|
+
reported = ""
|
|
91
|
+
if reported:
|
|
92
|
+
print(f" runs: {reported}")
|
|
93
|
+
expected = str(upstream).lstrip("v")
|
|
94
|
+
if expected and expected not in reported:
|
|
95
|
+
print(f" WARNING: binary reports {reported}, manifest pins {upstream}")
|
|
96
|
+
else:
|
|
97
|
+
print(" runs: NO — binary present but will not start")
|
|
98
|
+
|
|
99
|
+
print(f" hook script: {'present' if HOOK_FILE.is_file() else 'MISSING'}")
|
|
100
|
+
print(f" hook registered in settings.json: {'yes' if hook_registered() else 'no'}")
|
|
101
|
+
return 0
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
if __name__ == "__main__":
|
|
105
|
+
sys.exit(main())
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"last_run": "2026-07-
|
|
2
|
+
"last_run": "2026-07-26T11:36:57Z",
|
|
3
3
|
"schema_version": 1,
|
|
4
4
|
"tools": {
|
|
5
5
|
"aider": {
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
}
|
|
25
25
|
},
|
|
26
26
|
"augment": {
|
|
27
|
-
"docs_hash": "
|
|
27
|
+
"docs_hash": "1dd9e9f490e7103a",
|
|
28
28
|
"headings": [
|
|
29
29
|
"Admin",
|
|
30
30
|
"Auggie CLI",
|
|
@@ -66,7 +66,7 @@
|
|
|
66
66
|
}
|
|
67
67
|
},
|
|
68
68
|
"claude-app": {
|
|
69
|
-
"docs_hash": "
|
|
69
|
+
"docs_hash": "9a5cb995316c7c71",
|
|
70
70
|
"headings": [
|
|
71
71
|
"Add global and folder instructions",
|
|
72
72
|
"Availability",
|
|
@@ -107,7 +107,7 @@
|
|
|
107
107
|
}
|
|
108
108
|
},
|
|
109
109
|
"claude-code": {
|
|
110
|
-
"docs_hash": "
|
|
110
|
+
"docs_hash": "d373a3e4db1f4c79",
|
|
111
111
|
"headings": [
|
|
112
112
|
"Core concepts",
|
|
113
113
|
"Documentation Index",
|
|
@@ -163,10 +163,10 @@
|
|
|
163
163
|
"slash command": false,
|
|
164
164
|
"sub-agent": true
|
|
165
165
|
},
|
|
166
|
-
"version": "2.1.
|
|
166
|
+
"version": "2.1.220 (Claude Code)"
|
|
167
167
|
},
|
|
168
168
|
"cline": {
|
|
169
|
-
"docs_hash": "
|
|
169
|
+
"docs_hash": "d76db24ff58cb7ea",
|
|
170
170
|
"headings": [
|
|
171
171
|
"API Reference",
|
|
172
172
|
"Best Practices",
|
|
@@ -213,7 +213,7 @@
|
|
|
213
213
|
}
|
|
214
214
|
},
|
|
215
215
|
"codex-cli": {
|
|
216
|
-
"docs_hash": "
|
|
216
|
+
"docs_hash": "379d4d3760235c7f",
|
|
217
217
|
"headings": [
|
|
218
218
|
"API",
|
|
219
219
|
"API Reference",
|
|
@@ -243,12 +243,10 @@
|
|
|
243
243
|
"Connect tools and data",
|
|
244
244
|
"Connection methods",
|
|
245
245
|
"Contribute",
|
|
246
|
-
"Conversion
|
|
247
|
-
"Core Concepts",
|
|
246
|
+
"Conversion specs",
|
|
248
247
|
"Core concepts",
|
|
249
248
|
"Cost and throughput",
|
|
250
249
|
"Customization",
|
|
251
|
-
"Deploy",
|
|
252
250
|
"Deployment and model providers",
|
|
253
251
|
"Desktop app",
|
|
254
252
|
"Development workflows",
|
|
@@ -309,6 +307,7 @@
|
|
|
309
307
|
"Start your first task",
|
|
310
308
|
"Stay in control",
|
|
311
309
|
"Suggested",
|
|
310
|
+
"Test and publish",
|
|
312
311
|
"Text and code",
|
|
313
312
|
"Third-party integrations",
|
|
314
313
|
"Topics",
|
|
@@ -349,7 +348,7 @@
|
|
|
349
348
|
"version": "codex-cli 0.145.0"
|
|
350
349
|
},
|
|
351
350
|
"cursor": {
|
|
352
|
-
"docs_hash": "
|
|
351
|
+
"docs_hash": "74d1366066760874",
|
|
353
352
|
"headings": [],
|
|
354
353
|
"markers": {
|
|
355
354
|
".cursor/rules": false,
|
|
@@ -365,7 +364,7 @@
|
|
|
365
364
|
}
|
|
366
365
|
},
|
|
367
366
|
"gemini-cli": {
|
|
368
|
-
"docs_hash": "
|
|
367
|
+
"docs_hash": "c6d93617f9359329",
|
|
369
368
|
"headings": [
|
|
370
369
|
"Breadcrumbs",
|
|
371
370
|
"Directory actions",
|
|
@@ -408,7 +407,7 @@
|
|
|
408
407
|
}
|
|
409
408
|
},
|
|
410
409
|
"github-copilot": {
|
|
411
|
-
"docs_hash": "
|
|
410
|
+
"docs_hash": "1c6e0fd930f573fa",
|
|
412
411
|
"headings": [
|
|
413
412
|
"About Copilot auto model selection",
|
|
414
413
|
"About Copilot automations",
|
|
@@ -461,7 +460,7 @@
|
|
|
461
460
|
}
|
|
462
461
|
},
|
|
463
462
|
"opencode": {
|
|
464
|
-
"docs_hash": "
|
|
463
|
+
"docs_hash": "297813a7e2881afe",
|
|
465
464
|
"headings": [
|
|
466
465
|
"Add features",
|
|
467
466
|
"Ask questions",
|
|
@@ -521,7 +520,7 @@
|
|
|
521
520
|
}
|
|
522
521
|
},
|
|
523
522
|
"windsurf": {
|
|
524
|
-
"docs_hash": "
|
|
523
|
+
"docs_hash": "d57d379406804a4b",
|
|
525
524
|
"headings": [
|
|
526
525
|
"Accounts",
|
|
527
526
|
"Advanced",
|
package/bin/ai-toolkit.js
CHANGED
|
@@ -50,7 +50,6 @@ const SCRIPT_COMMANDS = {
|
|
|
50
50
|
'benchmark-ecosystem': { script: 'benchmark_ecosystem.py', toolkitCwd: true },
|
|
51
51
|
'evaluate': { script: 'evaluate_skills.py', toolkitCwd: true },
|
|
52
52
|
'stats': { script: 'stats.py' },
|
|
53
|
-
'output-filter': { script: 'output_filter_cli.py' },
|
|
54
53
|
'compile-slm': { script: 'compile_slm.py' },
|
|
55
54
|
'pack-codebase': { script: 'pack_codebase.py' },
|
|
56
55
|
'claude-app': { script: 'claude_app.py', toolkitCwd: true },
|
|
@@ -80,7 +79,6 @@ const COMMANDS = {
|
|
|
80
79
|
'benchmark-ecosystem': 'Generate ecosystem benchmark snapshot (GitHub metadata + offline fallback)',
|
|
81
80
|
evaluate: 'Run skill evaluation suite',
|
|
82
81
|
stats: 'Show skill usage statistics (--summary for product telemetry, --reset to clear)',
|
|
83
|
-
'output-filter': 'Manage native tool-output filter (status, inspect, recover, clean)',
|
|
84
82
|
create: 'Scaffold new skill from template (e.g. create skill my-lint --template=linter)',
|
|
85
83
|
mcp: 'Manage MCP templates and install native editor MCP configs',
|
|
86
84
|
config: 'Manage config inheritance (validate, diff, init, create-base, check)',
|
|
@@ -576,6 +574,21 @@ function handleUpdate(args) {
|
|
|
576
574
|
// User-provided args override state-derived args
|
|
577
575
|
run(scriptPath('install.py'), [...stateArgs, ...args]);
|
|
578
576
|
|
|
577
|
+
// Installed plugin packs live under ~/.softspark/ai-toolkit and are global,
|
|
578
|
+
// so --local (project-local config only) leaves them alone. The wiring is
|
|
579
|
+
// generic: plugin.py decides per pack whether anything changed and stays
|
|
580
|
+
// silent when nothing has, so this adds no noise for the common case.
|
|
581
|
+
if (!isLocal) {
|
|
582
|
+
const pluginArgs = ['update', '--editor', 'all', '--all'];
|
|
583
|
+
if (isDryRun) pluginArgs.push('--dry-run');
|
|
584
|
+
try {
|
|
585
|
+
run(scriptPath('plugin.py'), pluginArgs);
|
|
586
|
+
} catch (_err) {
|
|
587
|
+
// A pack update never fails the core update that already succeeded.
|
|
588
|
+
console.warn('WARN: plugin pack update skipped');
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
|
|
579
592
|
// After global update (not --local), propagate to all registered projects
|
|
580
593
|
if (!isLocal && !isDryRun) {
|
|
581
594
|
const registryPath = path.join(process.env.HOME, '.softspark', 'ai-toolkit', 'projects.json');
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Retirement: Native Tool-Output Filter — Measured 0% and Removed"
|
|
3
|
+
category: planning
|
|
4
|
+
service: ai-toolkit
|
|
5
|
+
tags:
|
|
6
|
+
- output-filter
|
|
7
|
+
- token-reduction
|
|
8
|
+
- postmortem
|
|
9
|
+
- measurement
|
|
10
|
+
- claude-code
|
|
11
|
+
doc_type: postmortem
|
|
12
|
+
status: completed
|
|
13
|
+
created: "2026-07-26"
|
|
14
|
+
last_updated: "2026-07-26"
|
|
15
|
+
shipped_in: "v4.17.0 (removal)"
|
|
16
|
+
description: "Why the native tool-output filter shipped in v4.16.0 was removed in v4.17.0: measured 0.0000% whole-session token saving on real traffic, because agent-issued commands are compound and the design accepted only simple registered shapes."
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
# Retirement: Native Tool-Output Filter
|
|
20
|
+
|
|
21
|
+
**Shipped:** v4.16.0 (2026-07-23). **Removed:** v4.17.0 (2026-07-26).
|
|
22
|
+
|
|
23
|
+
## The number
|
|
24
|
+
|
|
25
|
+
Measured whole-session input-token saving: **0.0000%**.
|
|
26
|
+
|
|
27
|
+
Real Bash results from local Claude Code transcripts were replayed through the
|
|
28
|
+
shipped classifier and the full filter registry. The filters ran on the actual
|
|
29
|
+
captured output; this is a measurement, not an estimate.
|
|
30
|
+
|
|
31
|
+
| Scope | Value |
|
|
32
|
+
|---|---:|
|
|
33
|
+
| Transcripts replayed | 134, across 22 distinct projects |
|
|
34
|
+
| Successful Bash results | 7600 |
|
|
35
|
+
| Of those, parsed as a simple command shape | 145 (1.9%) |
|
|
36
|
+
| Of those, matched a registered shape | 18 (0.24%) — 16 `git diff`, 2 `git show` |
|
|
37
|
+
| Accepted by any filter | **0** |
|
|
38
|
+
| Bytes saved | **0** |
|
|
39
|
+
|
|
40
|
+
The classifier was verified working before the result was accepted: `git
|
|
41
|
+
status`, `git log -n 20`, `pytest -v`, `bats --tap`, and `npm test` each
|
|
42
|
+
produced exactly one candidate. The zero is real.
|
|
43
|
+
|
|
44
|
+
## Why: the premise, not the implementation
|
|
45
|
+
|
|
46
|
+
Seventeen filters were correct. They cleared their byte floors on owned
|
|
47
|
+
fixtures (44–95% reduction), stayed inside every latency budget at the 8 MiB
|
|
48
|
+
engine cap, passed adversarial safety review, and never once compressed a
|
|
49
|
+
failure. None of that mattered, because the commands they were built for are
|
|
50
|
+
not the commands that get issued.
|
|
51
|
+
|
|
52
|
+
95% of successful Bash invocations are compound. The byte pool breaks down as:
|
|
53
|
+
|
|
54
|
+
| Class | Share of compound bytes | Why the filter refused it |
|
|
55
|
+
|---|---:|---|
|
|
56
|
+
| `;` chain | 44.3% | multiple output producers, attribution ambiguous |
|
|
57
|
+
| multiline script | 29.1% | rejected at the raw-string boundary |
|
|
58
|
+
| pipeline | 12.7% | the pipe transformed the output |
|
|
59
|
+
| `&&` chain with producing segments | 5.6% | multiple output producers |
|
|
60
|
+
| redirect, substitution | 4.4% | rejected at the raw-string boundary |
|
|
61
|
+
| heredoc | 3.5% | rejected at the raw-string boundary |
|
|
62
|
+
|
|
63
|
+
Every one of those refusals was the correct safety decision in isolation.
|
|
64
|
+
Together they excluded the entire population.
|
|
65
|
+
|
|
66
|
+
The most-frequent single shape was `cd <path> && …`, at 375 results and 480 KB.
|
|
67
|
+
A bounded `cd`-prefix subset had already been designed, threat-modelled, and
|
|
68
|
+
measured during Phase 3, and it was dropped because it covered 0.00% of the
|
|
69
|
+
compound pool. The retirement measurement confirms why: of those 375 results,
|
|
70
|
+
only 5 had a single simple second segment, and those 5 produced 0 bytes of
|
|
71
|
+
output.
|
|
72
|
+
|
|
73
|
+
## What was already rejected on the way, and still stands
|
|
74
|
+
|
|
75
|
+
- **Read-result coverage: rejected on the `Edit` exact-match hazard.** On
|
|
76
|
+
670.7 KB of real Read content, adjacent-duplicate collapse saves 0.00%,
|
|
77
|
+
blank-run collapse 0.04%, trailing-whitespace 0.00%. Anything above noise
|
|
78
|
+
requires elision, and 80.7% of `Edit` old-strings target a file read earlier
|
|
79
|
+
in the same session, 63.4% of them multi-line byte-exact quotes. A Read
|
|
80
|
+
result asserts what is on disk, so omission is a false claim rather than a
|
|
81
|
+
summary.
|
|
82
|
+
- **Compound-command subset: designed, measured at 0.00% coverage, dropped.**
|
|
83
|
+
The pipeline-truncator shape failed on an inversion: a truncated document
|
|
84
|
+
parses cleanly exactly where the shape would pay, and rejects exactly where
|
|
85
|
+
truncation is detectable.
|
|
86
|
+
|
|
87
|
+
## The process lesson
|
|
88
|
+
|
|
89
|
+
The plan validated its **design** exhaustively across five phases and its
|
|
90
|
+
**premise** not at all until the fifth. Fixtures measured the filter; only real
|
|
91
|
+
traffic measured the value, and the two disagreed by two orders of magnitude.
|
|
92
|
+
|
|
93
|
+
The end-to-end replay that produced the 0% took under an hour and could have
|
|
94
|
+
run on day one, before any filter existed. Any future plan of this shape must
|
|
95
|
+
put premise validation in Phase 0, with a kill number published before the
|
|
96
|
+
measurement rather than argued after it.
|
|
97
|
+
|
|
98
|
+
## Evaluated as a replacement: rtk
|
|
99
|
+
|
|
100
|
+
`rtk` (https://github.com/rtk-ai/rtk, Apache-2.0) rewrites commands at
|
|
101
|
+
`PreToolUse` rather than filtering output afterwards, which is the mechanism
|
|
102
|
+
this project's own safety contract had excluded. Its rewrite pipeline was
|
|
103
|
+
ported and validated against 197 of its own test assertions (197/197 exact
|
|
104
|
+
agreement), then applied to the same traffic:
|
|
105
|
+
|
|
106
|
+
- addresses **31.5%** of successful Bash bytes, **9.7%** of all tool-result
|
|
107
|
+
bytes — genuinely non-zero, so the in-house 0% was a coverage failure rather
|
|
108
|
+
than a law of nature;
|
|
109
|
+
- projected saving is **0.32–0.48%** of session input tokens on rtk's own
|
|
110
|
+
60–90% claim, and **0.15–0.21%** once its filters' actual behaviour is
|
|
111
|
+
modelled;
|
|
112
|
+
- its two largest families here under-deliver: `rtk read` returns files
|
|
113
|
+
verbatim at the default `--level none`, and `rtk grep` models at 12.3%
|
|
114
|
+
against a claimed 75%;
|
|
115
|
+
- custom TOML filters, the documented extension point, would reach 1.91% of
|
|
116
|
+
Bash bytes. The large misses are structurally unreachable from config:
|
|
117
|
+
`| head` and `| tail` (34.6%) are blocked by the pipeline-final rule, and
|
|
118
|
+
`sed` (19.4%) sits in the hard-ignored prefix list.
|
|
119
|
+
|
|
120
|
+
Not adopted.
|
|
121
|
+
|
|
122
|
+
## Where the tokens actually are
|
|
123
|
+
|
|
124
|
+
The measurement points somewhere other than command output. In this traffic,
|
|
125
|
+
`Read` is 53.8% of tool-result bytes, and within Bash the two largest buckets
|
|
126
|
+
are `sed` used as a file reader (16.2%) and `| head` / `| tail` pipeline tails
|
|
127
|
+
(34.6%). Those are file-reading patterns, not tool reports. Any future attempt
|
|
128
|
+
at token reduction should start there, and should start by measuring.
|