@softspark/ai-toolkit 4.16.1 → 4.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +84 -0
- package/README.md +13 -19
- package/app/.claude-plugin/plugin.json +1 -1
- package/app/hooks/session-end.sh +1 -13
- package/app/hooks.json +0 -10
- package/app/plugins/README.md +16 -4
- package/app/plugins/rtk-pack/README.md +123 -0
- package/app/plugins/rtk-pack/hooks/rewrite.sh +79 -0
- package/app/plugins/rtk-pack/plugin.json +60 -0
- package/app/plugins/rtk-pack/scripts/init.py +252 -0
- package/app/plugins/rtk-pack/scripts/status.py +105 -0
- package/benchmarks/ecosystem-doctor-snapshot.json +14 -15
- package/bin/ai-toolkit.js +15 -2
- package/kb/history/completed/output-filter-retirement-20260726.md +128 -0
- package/kb/history/completed/rtk-pack-integration-20260726.md +710 -0
- package/kb/procedures/maintenance-sop.md +1 -1
- package/kb/procedures/release-preparation-sop.md +8 -3
- package/kb/procedures/rtk-upstream-sync-sop.md +279 -0
- package/kb/reference/architecture-overview.md +2 -3
- package/kb/reference/cli-reference.md +4 -14
- package/kb/reference/enterprise-config-guide.md +1 -21
- package/kb/reference/hooks-catalog.md +3 -60
- package/kb/reference/plugin-pack-conventions.md +17 -2
- package/kb/reference/supported-tools-registry.md +0 -4
- package/llms-full.txt +1171 -402
- package/llms.txt +3 -1
- package/manifest.json +147 -36
- package/package.json +1 -2
- package/scripts/audit_skills.py +21 -0
- package/scripts/claude_app.py +2 -21
- package/scripts/config_cli.py +4 -0
- package/scripts/config_merger.py +0 -17
- package/scripts/config_validator.py +11 -138
- package/scripts/doctor.py +3 -20
- package/scripts/install.py +2 -1
- package/scripts/install_steps/ai_tools.py +28 -99
- package/scripts/install_steps/hooks.py +26 -24
- package/scripts/merge-hooks.py +33 -2
- package/scripts/output_filter_retirement.py +395 -0
- package/scripts/plugin.py +136 -16
- package/scripts/schemas/ai-toolkit-config.schema.json +0 -60
- package/scripts/uninstall.py +13 -27
- package/scripts/verify_rtk_binary.py +335 -0
- package/app/hooks/filter-tool-output.sh +0 -76
- package/app/output-filter-policy.json +0 -15
- package/benchmarks/output-filter/README.md +0 -11
- package/benchmarks/output-filter/scenarios.json +0 -25
- package/kb/reference/tool-output-filter.md +0 -288
- package/scripts/benchmark_output_filter.py +0 -343
- package/scripts/output_filter_cli.py +0 -347
- package/scripts/output_filter_hook.py +0 -23
- package/scripts/tool_output_filter/__init__.py +0 -33
- package/scripts/tool_output_filter/contracts.py +0 -173
- package/scripts/tool_output_filter/engine.py +0 -260
- package/scripts/tool_output_filter/hook_runtime.py +0 -369
- package/scripts/tool_output_filter/input.py +0 -56
- package/scripts/tool_output_filter/invariants.py +0 -40
- package/scripts/tool_output_filter/policy.py +0 -153
- package/scripts/tool_output_filter/profiles/__init__.py +0 -68
- package/scripts/tool_output_filter/profiles/repeat_lines.py +0 -71
- package/scripts/tool_output_filter/profiles/tap_success.py +0 -154
- package/scripts/tool_output_filter/recovery.py +0 -846
- package/scripts/tool_output_filter/telemetry.py +0 -13
|
@@ -39,66 +39,6 @@
|
|
|
39
39
|
"default": "standard",
|
|
40
40
|
"description": "Installation profile controlling which modules are installed."
|
|
41
41
|
},
|
|
42
|
-
"toolOutputFilter": {
|
|
43
|
-
"type": "object",
|
|
44
|
-
"additionalProperties": false,
|
|
45
|
-
"description": "Native post-execution Bash output filtering. Disabled by default.",
|
|
46
|
-
"properties": {
|
|
47
|
-
"mode": {
|
|
48
|
-
"type": "string",
|
|
49
|
-
"enum": ["off", "observe", "safe"],
|
|
50
|
-
"default": "off"
|
|
51
|
-
},
|
|
52
|
-
"profiles": {
|
|
53
|
-
"type": "array",
|
|
54
|
-
"items": {
|
|
55
|
-
"type": "string",
|
|
56
|
-
"enum": ["repeat-lines", "tap-success"]
|
|
57
|
-
},
|
|
58
|
-
"uniqueItems": true,
|
|
59
|
-
"default": ["repeat-lines", "tap-success"]
|
|
60
|
-
},
|
|
61
|
-
"maxInputBytes": {
|
|
62
|
-
"type": "integer",
|
|
63
|
-
"minimum": 1,
|
|
64
|
-
"maximum": 8388608,
|
|
65
|
-
"default": 8388608
|
|
66
|
-
},
|
|
67
|
-
"minSavingsBytes": {
|
|
68
|
-
"type": "integer",
|
|
69
|
-
"minimum": 0,
|
|
70
|
-
"maximum": 8388608,
|
|
71
|
-
"default": 1024
|
|
72
|
-
},
|
|
73
|
-
"minSavingsRatio": {
|
|
74
|
-
"type": "number",
|
|
75
|
-
"minimum": 0,
|
|
76
|
-
"maximum": 1,
|
|
77
|
-
"default": 0.15
|
|
78
|
-
},
|
|
79
|
-
"recovery": {
|
|
80
|
-
"type": "object",
|
|
81
|
-
"additionalProperties": false,
|
|
82
|
-
"properties": {
|
|
83
|
-
"mode": {
|
|
84
|
-
"type": "string",
|
|
85
|
-
"enum": ["ephemeral"],
|
|
86
|
-
"default": "ephemeral"
|
|
87
|
-
},
|
|
88
|
-
"ttlMinutes": {
|
|
89
|
-
"type": "integer",
|
|
90
|
-
"minimum": 1,
|
|
91
|
-
"default": 60
|
|
92
|
-
},
|
|
93
|
-
"maxSessionBytes": {
|
|
94
|
-
"type": "integer",
|
|
95
|
-
"minimum": 1,
|
|
96
|
-
"default": 33554432
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
},
|
|
102
42
|
"agents": {
|
|
103
43
|
"type": "object",
|
|
104
44
|
"additionalProperties": false,
|
package/scripts/uninstall.py
CHANGED
|
@@ -5,8 +5,8 @@ The default scope is the current user's global install. ``--local`` targets a
|
|
|
5
5
|
project, while an explicit legacy positional target scans both project and
|
|
6
6
|
home-style locations for backward compatibility. Only files, symlinks, JSON
|
|
7
7
|
handlers, and marker blocks with verifiable ai-toolkit ownership are removed.
|
|
8
|
-
Global scope also removes validated
|
|
9
|
-
foreign content in the same session trees.
|
|
8
|
+
Global scope also removes validated recovery files left behind by the v4.16.x
|
|
9
|
+
tool-output filter while keeping foreign content in the same session trees.
|
|
10
10
|
|
|
11
11
|
Usage:
|
|
12
12
|
python3 scripts/uninstall.py [--yes] [--local|--global] [--target DIR]
|
|
@@ -33,8 +33,16 @@ from typing import Any
|
|
|
33
33
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
34
34
|
from _common import app_dir, toolkit_dir
|
|
35
35
|
from injection import strip_all_sections, strip_section, trim_trailing_blanks
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
# Retirement cleanup for the v4.16.x tool-output filter. The runtime package
|
|
37
|
+
# that wrote those files is gone; output_filter_retirement re-states its
|
|
38
|
+
# ownership rules as self-contained constants for both install and uninstall.
|
|
39
|
+
from output_filter_retirement import (
|
|
40
|
+
PROJECT_OWNER_NAME as _OUTPUT_FILTER_OWNER_NAME,
|
|
41
|
+
PROJECT_POLICY_NAME as _OUTPUT_FILTER_POLICY_NAME,
|
|
42
|
+
clean_owned_recovery_tree,
|
|
43
|
+
count_owned_recovery_artifacts,
|
|
44
|
+
managed_project_policy as _managed_output_filter_policy,
|
|
45
|
+
)
|
|
38
46
|
|
|
39
47
|
|
|
40
48
|
CODEX_AGENT_MARKER = "# ai-toolkit-managed: codex-agent"
|
|
@@ -559,28 +567,6 @@ def _discover_claude_hooks(claude_dir: Path) -> list[tuple[str, str]]:
|
|
|
559
567
|
return []
|
|
560
568
|
|
|
561
569
|
|
|
562
|
-
_OUTPUT_FILTER_POLICY_NAME = "ai-toolkit-output-filter.json"
|
|
563
|
-
_OUTPUT_FILTER_OWNER_NAME = ".ai-toolkit-output-filter.owner"
|
|
564
|
-
_OUTPUT_FILTER_OWNER_MARKER = b"ai-toolkit-output-filter-policy-v1\n"
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
def _managed_output_filter_policy(claude_dir: Path) -> list[Path]:
|
|
568
|
-
"""Return the managed per-project policy pair, or [] when unmanaged."""
|
|
569
|
-
policy = claude_dir / _OUTPUT_FILTER_POLICY_NAME
|
|
570
|
-
owner = claude_dir / _OUTPUT_FILTER_OWNER_NAME
|
|
571
|
-
if owner.is_symlink() or not owner.is_file():
|
|
572
|
-
return []
|
|
573
|
-
try:
|
|
574
|
-
if owner.read_bytes() != _OUTPUT_FILTER_OWNER_MARKER:
|
|
575
|
-
return []
|
|
576
|
-
except OSError:
|
|
577
|
-
return []
|
|
578
|
-
managed = [owner]
|
|
579
|
-
if not policy.is_symlink() and policy.is_file():
|
|
580
|
-
managed.insert(0, policy)
|
|
581
|
-
return managed
|
|
582
|
-
|
|
583
|
-
|
|
584
570
|
def _discover_output_filter_policy(claude_dir: Path) -> list[tuple[str, str]]:
|
|
585
571
|
if not _managed_output_filter_policy(claude_dir):
|
|
586
572
|
return []
|
|
@@ -1318,7 +1304,7 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
|
|
|
1318
1304
|
parser = argparse.ArgumentParser(
|
|
1319
1305
|
description=(
|
|
1320
1306
|
"Remove only ai-toolkit-managed Claude, Codex, Copilot, and "
|
|
1321
|
-
"
|
|
1307
|
+
"leftover v4.16.x recovery data while preserving user-owned content."
|
|
1322
1308
|
),
|
|
1323
1309
|
epilog=(
|
|
1324
1310
|
"Global Codex and Copilot locations honor CODEX_HOME and "
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Verify a cross-built rtk binary is silent, and fingerprint it for drift.
|
|
3
|
+
|
|
4
|
+
Phase 1 of kb/history/completed/rtk-pack-integration-20260726.md. "No telemetry symbols in
|
|
5
|
+
the binary" is NOT a usable acceptance test: the guard is a runtime branch on a
|
|
6
|
+
const (`telemetry.rs:23-26`), not a `#[cfg]`, and `Cargo.toml:51` sets
|
|
7
|
+
`strip = true`, so a symbol check passes for the wrong reason. This script
|
|
8
|
+
asserts what is actually checkable instead:
|
|
9
|
+
|
|
10
|
+
build-gate RTK_TELEMETRY_URL / RTK_TELEMETRY_TOKEN unset at build time
|
|
11
|
+
runs the binary starts and reports its version
|
|
12
|
+
no-state a sandboxed run creates no telemetry state on disk
|
|
13
|
+
offline a run with no network route behaves identically (Linux only)
|
|
14
|
+
fingerprint size, digest and TLS-marker scan, recorded for drift detection
|
|
15
|
+
|
|
16
|
+
Stdlib only. Emits JSON to stdout; exits non-zero if any assertion fails.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import platform
|
|
25
|
+
import re
|
|
26
|
+
import shutil
|
|
27
|
+
import subprocess
|
|
28
|
+
import sys
|
|
29
|
+
import tempfile
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
# Compile-time telemetry inputs. Upstream injects these in its own release
|
|
33
|
+
# workflow (release.yml:85-86); our builds must leave both undefined.
|
|
34
|
+
TELEMETRY_BUILD_VARS = ("RTK_TELEMETRY_URL", "RTK_TELEMETRY_TOKEN")
|
|
35
|
+
|
|
36
|
+
# Telemetry state rtk would create if it ever ran the ping path. Names come from
|
|
37
|
+
# src/core/telemetry.rs (salt file) and the tracking DB under the data dir.
|
|
38
|
+
TELEMETRY_STATE_GLOBS = ("**/rtk/*salt*", "**/rtk/telemetry*", "**/rtk/*consent*")
|
|
39
|
+
|
|
40
|
+
# webpki-roots embeds CA subjects as readable DER. Their presence means rustls
|
|
41
|
+
# survived LTO, which is advisory rather than a failure: it says the HTTP stack
|
|
42
|
+
# was linked in, not that anything is sent. Recorded so drift is visible.
|
|
43
|
+
TLS_MARKERS = (b"ISRG Root X1", b"DigiCert", b"Baltimore CyberTrust", b"GlobalSign")
|
|
44
|
+
|
|
45
|
+
RUNNABLE_HERE = {
|
|
46
|
+
("Darwin", "arm64"): {"aarch64-apple-darwin"},
|
|
47
|
+
("Darwin", "x86_64"): {"x86_64-apple-darwin"},
|
|
48
|
+
("Linux", "x86_64"): {"x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl"},
|
|
49
|
+
("Linux", "aarch64"): {"aarch64-unknown-linux-gnu", "aarch64-unknown-linux-musl"},
|
|
50
|
+
("Windows", "AMD64"): {"x86_64-pc-windows-msvc"},
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
# qemu-user turns the one genuinely cross-built target into a verifiable one.
|
|
54
|
+
# Without it every assertion below reports "skipped" and the artifact ships
|
|
55
|
+
# having been started exactly zero times.
|
|
56
|
+
QEMU_FOR = {
|
|
57
|
+
"aarch64-unknown-linux-gnu": ("qemu-aarch64-static", "qemu-aarch64"),
|
|
58
|
+
"aarch64-unknown-linux-musl": ("qemu-aarch64-static", "qemu-aarch64"),
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
# Rosetta 2 does the same job on Apple silicon, which is what lets us verify an
|
|
62
|
+
# x86_64 artifact built on an arm64 runner. Probed, never assumed: the image can
|
|
63
|
+
# ship without it.
|
|
64
|
+
ROSETTA_FOR = {"x86_64-apple-darwin": ("Darwin", "arm64")}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Failure(Exception):
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def can_run_natively(target: str) -> bool:
|
|
72
|
+
return target in RUNNABLE_HERE.get((platform.system(), platform.machine()), set())
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def emulator_for(target: str) -> list | None:
|
|
76
|
+
"""qemu invocation for this target, with the sysroot passed as a flag.
|
|
77
|
+
|
|
78
|
+
The prefix goes through `-L` rather than QEMU_LD_PREFIX because the offline
|
|
79
|
+
check runs under sudo, and sudo's env_reset strips the variable. A gnu
|
|
80
|
+
target is dynamically linked, so losing it means qemu cannot find
|
|
81
|
+
ld-linux-aarch64.so.1 and the process dies with 255 before main.
|
|
82
|
+
"""
|
|
83
|
+
if can_run_natively(target) or platform.system() != "Linux":
|
|
84
|
+
return None
|
|
85
|
+
for candidate in QEMU_FOR.get(target, ()):
|
|
86
|
+
found = shutil.which(candidate)
|
|
87
|
+
if not found:
|
|
88
|
+
continue
|
|
89
|
+
prefix = os.environ.get("QEMU_LD_PREFIX", "")
|
|
90
|
+
return [found, "-L", prefix] if prefix else [found]
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def rosetta_for(target: str) -> list | None:
|
|
95
|
+
if ROSETTA_FOR.get(target) != (platform.system(), platform.machine()):
|
|
96
|
+
return None
|
|
97
|
+
if shutil.which("arch") is None:
|
|
98
|
+
return None
|
|
99
|
+
try:
|
|
100
|
+
probe = subprocess.run(
|
|
101
|
+
["arch", "-x86_64", "/usr/bin/true"], capture_output=True, timeout=30
|
|
102
|
+
)
|
|
103
|
+
except (OSError, subprocess.SubprocessError):
|
|
104
|
+
return None
|
|
105
|
+
return ["arch", "-x86_64"] if probe.returncode == 0 else None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def launcher(target: str):
|
|
109
|
+
"""(wrapper, how) for running this target here, or (None, reason)."""
|
|
110
|
+
if can_run_natively(target):
|
|
111
|
+
return [], "native"
|
|
112
|
+
emu = emulator_for(target)
|
|
113
|
+
if emu:
|
|
114
|
+
return emu, f"emulated via {Path(emu[0]).name}"
|
|
115
|
+
rosetta = rosetta_for(target)
|
|
116
|
+
if rosetta:
|
|
117
|
+
return rosetta, "translated via Rosetta 2"
|
|
118
|
+
return None, f"{target} is not runnable on {platform.system()}/{platform.machine()} and no emulator is installed"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def sandbox_env(root: Path) -> dict:
|
|
122
|
+
"""An environment whose config/data/cache all resolve inside root."""
|
|
123
|
+
env = dict(os.environ)
|
|
124
|
+
for var in TELEMETRY_BUILD_VARS:
|
|
125
|
+
env.pop(var, None)
|
|
126
|
+
env["HOME"] = str(root)
|
|
127
|
+
env["USERPROFILE"] = str(root)
|
|
128
|
+
env["XDG_CONFIG_HOME"] = str(root / "config")
|
|
129
|
+
env["XDG_DATA_HOME"] = str(root / "data")
|
|
130
|
+
env["XDG_CACHE_HOME"] = str(root / "cache")
|
|
131
|
+
env["APPDATA"] = str(root / "AppData" / "Roaming")
|
|
132
|
+
env["LOCALAPPDATA"] = str(root / "AppData" / "Local")
|
|
133
|
+
return env
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def run(binary: Path, args: list, env: dict, wrapper: list | None = None):
|
|
137
|
+
cmd = (wrapper or []) + [str(binary)] + args
|
|
138
|
+
proc = subprocess.run(cmd, env=env, capture_output=True, timeout=120)
|
|
139
|
+
return proc.returncode, proc.stdout.decode("utf-8", "replace"), proc.stderr.decode("utf-8", "replace")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def check_build_gate() -> dict:
|
|
143
|
+
"""The variables must be undefined in this environment too.
|
|
144
|
+
|
|
145
|
+
The workflow asserts this before `cargo build`; re-asserting here catches a
|
|
146
|
+
verification job that was handed a binary from a contaminated build.
|
|
147
|
+
"""
|
|
148
|
+
leaked = [v for v in TELEMETRY_BUILD_VARS if os.environ.get(v)]
|
|
149
|
+
if leaked:
|
|
150
|
+
raise Failure(f"telemetry build variables are set: {', '.join(leaked)}")
|
|
151
|
+
return {"pass": True, "checked": list(TELEMETRY_BUILD_VARS)}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def check_runs(binary: Path, target: str, upstream_tag: str) -> dict:
|
|
155
|
+
"""Starts, identifies itself as rtk, and reports the version we asked for.
|
|
156
|
+
|
|
157
|
+
Without the identity assertion this check passes for any binary that exits
|
|
158
|
+
0 on an unknown flag, `/bin/echo` included.
|
|
159
|
+
"""
|
|
160
|
+
wrapper, how = launcher(target)
|
|
161
|
+
if wrapper is None:
|
|
162
|
+
return {"pass": None, "skipped": how}
|
|
163
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
164
|
+
code, out, err = run(binary, ["--version"], sandbox_env(Path(tmp)), wrapper=wrapper)
|
|
165
|
+
if code != 0:
|
|
166
|
+
raise Failure(f"`rtk --version` exited {code}: {err.strip()[:200]}")
|
|
167
|
+
version = (out.strip() or err.strip())
|
|
168
|
+
if "rtk" not in version.lower():
|
|
169
|
+
raise Failure(f"`--version` output does not identify rtk: {version[:120]!r}")
|
|
170
|
+
expected = upstream_tag.lstrip("v")
|
|
171
|
+
if expected and expected not in version:
|
|
172
|
+
raise Failure(f"version {version[:120]!r} does not match upstream tag {upstream_tag}")
|
|
173
|
+
return {"pass": True, "version": version, "how": how}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def check_no_state(binary: Path, target: str) -> dict:
|
|
177
|
+
"""A real command must not leave telemetry state behind."""
|
|
178
|
+
wrapper, how = launcher(target)
|
|
179
|
+
if wrapper is None:
|
|
180
|
+
return {"pass": None, "skipped": how}
|
|
181
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
182
|
+
root = Path(tmp)
|
|
183
|
+
env = sandbox_env(root)
|
|
184
|
+
for args in (["--version"], ["--help"], ["git", "status"]):
|
|
185
|
+
run(binary, args, env, wrapper=wrapper)
|
|
186
|
+
found = sorted(
|
|
187
|
+
str(p.relative_to(root))
|
|
188
|
+
for pattern in TELEMETRY_STATE_GLOBS
|
|
189
|
+
for p in root.glob(pattern)
|
|
190
|
+
if p.is_file()
|
|
191
|
+
)
|
|
192
|
+
if found:
|
|
193
|
+
raise Failure(f"telemetry state created: {found}")
|
|
194
|
+
return {"pass": True, "sandbox_clean": True}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def check_offline(binary: Path, target: str) -> dict:
|
|
198
|
+
"""Run with no network route and assert identical behaviour.
|
|
199
|
+
|
|
200
|
+
`unshare -rn` gives a network namespace with only a down loopback, so any
|
|
201
|
+
outbound connection fails immediately. There is no equivalent that works
|
|
202
|
+
unprivileged on macOS or Windows runners, so this assertion is Linux-only
|
|
203
|
+
and reports itself skipped elsewhere rather than pretending to pass.
|
|
204
|
+
"""
|
|
205
|
+
wrapper, how = launcher(target)
|
|
206
|
+
if wrapper is None:
|
|
207
|
+
return {"pass": None, "skipped": how}
|
|
208
|
+
if platform.system() != "Linux" or shutil.which("unshare") is None:
|
|
209
|
+
return {"pass": None, "skipped": "unshare(1) network namespaces are Linux-only"}
|
|
210
|
+
|
|
211
|
+
# Ubuntu 24.04 sets kernel.apparmor_restrict_unprivileged_userns=1, so the
|
|
212
|
+
# unprivileged form is refused on GitHub runners and this assertion silently
|
|
213
|
+
# became a skip. Fall back to passwordless sudo, which runners have.
|
|
214
|
+
#
|
|
215
|
+
# Each entry is (baseline, isolated): identical except for the network
|
|
216
|
+
# namespace. Comparing against a plain run instead would confound the
|
|
217
|
+
# network with sudo's env_reset, and the difference would be read as
|
|
218
|
+
# evidence about the binary when it is evidence about the harness.
|
|
219
|
+
isolators = [
|
|
220
|
+
(["unshare", "-r"], ["unshare", "-rn"]),
|
|
221
|
+
(["sudo", "-n", "unshare", "-r"], ["sudo", "-n", "unshare", "-rn"]),
|
|
222
|
+
]
|
|
223
|
+
baseline = isolated = None
|
|
224
|
+
for base, iso in isolators:
|
|
225
|
+
if subprocess.run(iso + ["true"], capture_output=True, timeout=30).returncode == 0:
|
|
226
|
+
baseline, isolated = base, iso
|
|
227
|
+
break
|
|
228
|
+
if isolated is None:
|
|
229
|
+
return {"pass": None, "skipped": "no usable network namespace: unprivileged userns refused and sudo unavailable"}
|
|
230
|
+
|
|
231
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
232
|
+
env = sandbox_env(Path(tmp))
|
|
233
|
+
online_code, _, online_err = run(binary, ["--version"], env, wrapper=baseline + wrapper)
|
|
234
|
+
offline_code, _, offline_err = run(binary, ["--version"], env, wrapper=isolated + wrapper)
|
|
235
|
+
if online_code != 0:
|
|
236
|
+
# The baseline could not start inside the namespace, so the comparison
|
|
237
|
+
# says nothing about the binary. That is a harness limitation, not a
|
|
238
|
+
# defect in the artifact, and reporting it as a failure would blame the
|
|
239
|
+
# thing being measured for the measurement not working. Skip instead,
|
|
240
|
+
# and carry the reason so it is visible rather than silent.
|
|
241
|
+
return {
|
|
242
|
+
"pass": None,
|
|
243
|
+
"skipped": (
|
|
244
|
+
f"baseline run under {' '.join(baseline)} exited {online_code}, "
|
|
245
|
+
f"so the network comparison proves nothing"
|
|
246
|
+
),
|
|
247
|
+
"detail": online_err.strip()[:200],
|
|
248
|
+
}
|
|
249
|
+
if offline_code != online_code:
|
|
250
|
+
raise Failure(
|
|
251
|
+
f"behaviour differs without a network route: online exit {online_code}, offline exit {offline_code}"
|
|
252
|
+
)
|
|
253
|
+
return {
|
|
254
|
+
"pass": True,
|
|
255
|
+
"exit_code": offline_code,
|
|
256
|
+
"stderr_empty": not offline_err.strip(),
|
|
257
|
+
"isolator": " ".join(isolated),
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def fingerprint(binary: Path) -> dict:
|
|
262
|
+
data = binary.read_bytes()
|
|
263
|
+
markers = sorted(m.decode() for m in TLS_MARKERS if m in data)
|
|
264
|
+
printable = re.findall(rb"[\x20-\x7e]{8,}", data)
|
|
265
|
+
return {
|
|
266
|
+
"size_bytes": len(data),
|
|
267
|
+
"sha256": hashlib.sha256(data).hexdigest(),
|
|
268
|
+
"tls_markers_present": markers,
|
|
269
|
+
"printable_string_count": len(printable),
|
|
270
|
+
"strings_digest": hashlib.sha256(b"\n".join(sorted(set(printable)))).hexdigest(),
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def main() -> int:
|
|
275
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
276
|
+
ap.add_argument("--binary", required=True, type=Path)
|
|
277
|
+
ap.add_argument("--target", required=True)
|
|
278
|
+
ap.add_argument("--upstream-tag", default="", help="recorded in the manifest for traceability")
|
|
279
|
+
ap.add_argument("--out", type=Path, help="write the manifest here as well as stdout")
|
|
280
|
+
args = ap.parse_args()
|
|
281
|
+
|
|
282
|
+
if not args.binary.is_file():
|
|
283
|
+
print(json.dumps({"error": f"no such binary: {args.binary}"}), file=sys.stderr)
|
|
284
|
+
return 2
|
|
285
|
+
|
|
286
|
+
checks = {}
|
|
287
|
+
failures = []
|
|
288
|
+
for name, fn in (
|
|
289
|
+
("build_gate", lambda: check_build_gate()),
|
|
290
|
+
("runs", lambda: check_runs(args.binary, args.target, args.upstream_tag)),
|
|
291
|
+
("no_state", lambda: check_no_state(args.binary, args.target)),
|
|
292
|
+
("offline", lambda: check_offline(args.binary, args.target)),
|
|
293
|
+
):
|
|
294
|
+
try:
|
|
295
|
+
checks[name] = fn()
|
|
296
|
+
except Failure as exc:
|
|
297
|
+
checks[name] = {"pass": False, "reason": str(exc)}
|
|
298
|
+
failures.append(f"{name}: {exc}")
|
|
299
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
300
|
+
checks[name] = {"pass": False, "reason": f"{type(exc).__name__}: {exc}"}
|
|
301
|
+
failures.append(f"{name}: {exc}")
|
|
302
|
+
|
|
303
|
+
# A target nobody could start here must not report "pass". Every runtime
|
|
304
|
+
# assertion would have been skipped, and a green tick on an artifact that
|
|
305
|
+
# was never executed is the same silent-pass trap this plan keeps finding.
|
|
306
|
+
ran_anything = checks.get("runs", {}).get("pass") is True
|
|
307
|
+
if failures:
|
|
308
|
+
verdict = "fail"
|
|
309
|
+
elif ran_anything:
|
|
310
|
+
verdict = "pass"
|
|
311
|
+
else:
|
|
312
|
+
verdict = "inconclusive"
|
|
313
|
+
|
|
314
|
+
manifest = {
|
|
315
|
+
"target": args.target,
|
|
316
|
+
"upstream_tag": args.upstream_tag,
|
|
317
|
+
"host": f"{platform.system()}/{platform.machine()}",
|
|
318
|
+
# Recorded because the emulated path only works when the workflow
|
|
319
|
+
# supplies the cross sysroot; a manifest that does not say so cannot be
|
|
320
|
+
# audited later.
|
|
321
|
+
"qemu_ld_prefix": os.environ.get("QEMU_LD_PREFIX", ""),
|
|
322
|
+
"checks": checks,
|
|
323
|
+
"fingerprint": fingerprint(args.binary),
|
|
324
|
+
"verdict": verdict,
|
|
325
|
+
"failures": failures,
|
|
326
|
+
}
|
|
327
|
+
text = json.dumps(manifest, indent=2, sort_keys=True)
|
|
328
|
+
print(text)
|
|
329
|
+
if args.out:
|
|
330
|
+
args.out.write_text(text + "\n", encoding="utf-8")
|
|
331
|
+
return 1 if failures else 0
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
if __name__ == "__main__":
|
|
335
|
+
sys.exit(main())
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# Claude PostToolUse adapter for the native tool-output filter.
|
|
3
|
-
|
|
4
|
-
HOOK_SOURCE="${BASH_SOURCE[0]}"
|
|
5
|
-
HOOK_DIR="${HOOK_SOURCE%/*}"
|
|
6
|
-
[[ "$HOOK_DIR" == "$HOOK_SOURCE" ]] && HOOK_DIR="."
|
|
7
|
-
# shellcheck source=_profile-check.sh
|
|
8
|
-
source "$HOOK_DIR/_profile-check.sh"
|
|
9
|
-
|
|
10
|
-
OWNER_MARKER="ai-toolkit-output-filter-policy-v1"
|
|
11
|
-
GLOBAL_POLICY="$HOME/.softspark/ai-toolkit/hooks/output-filter-policy.json"
|
|
12
|
-
PROJECTS_REGISTRY="$HOME/.softspark/ai-toolkit/projects.json"
|
|
13
|
-
|
|
14
|
-
is_regular_file() {
|
|
15
|
-
[[ -f "$1" && -r "$1" && ! -L "$1" ]]
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
# A project policy is trusted only for projects the user registered via
|
|
19
|
-
# `ai-toolkit install --local`. The owner marker alone is a public constant,
|
|
20
|
-
# so a cloned repo must never be able to self-enable filtering with it.
|
|
21
|
-
is_registered_project() {
|
|
22
|
-
is_regular_file "$PROJECTS_REGISTRY" &&
|
|
23
|
-
grep -qF "\"$1\"" "$PROJECTS_REGISTRY" 2>/dev/null
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
if [[ "${AI_TOOLKIT_OUTPUT_FILTER_DISABLE:-}" == "1" ]]; then
|
|
27
|
-
exit 0
|
|
28
|
-
fi
|
|
29
|
-
|
|
30
|
-
if [[ -n "${AI_TOOLKIT_OUTPUT_FILTER_POLICY:-}" ]]; then
|
|
31
|
-
POLICY_PATH="$AI_TOOLKIT_OUTPUT_FILTER_POLICY"
|
|
32
|
-
if ! is_regular_file "$POLICY_PATH"; then
|
|
33
|
-
exit 0
|
|
34
|
-
fi
|
|
35
|
-
else
|
|
36
|
-
PROJECT_ROOT="${CLAUDE_PROJECT_DIR:-$PWD}"
|
|
37
|
-
PROJECT_POLICY="$PROJECT_ROOT/.claude/ai-toolkit-output-filter.json"
|
|
38
|
-
PROJECT_OWNER="$PROJECT_ROOT/.claude/.ai-toolkit-output-filter.owner"
|
|
39
|
-
if [[ -L "$PROJECT_ROOT" || -L "$PROJECT_ROOT/.claude" ]]; then
|
|
40
|
-
exit 0
|
|
41
|
-
fi
|
|
42
|
-
if is_registered_project "$PROJECT_ROOT" &&
|
|
43
|
-
is_regular_file "$PROJECT_OWNER" &&
|
|
44
|
-
[[ "$(<"$PROJECT_OWNER")" == "$OWNER_MARKER" ]]; then
|
|
45
|
-
if ! is_regular_file "$PROJECT_POLICY"; then
|
|
46
|
-
exit 0
|
|
47
|
-
fi
|
|
48
|
-
POLICY_PATH="$PROJECT_POLICY"
|
|
49
|
-
else
|
|
50
|
-
POLICY_PATH="$GLOBAL_POLICY"
|
|
51
|
-
fi
|
|
52
|
-
fi
|
|
53
|
-
|
|
54
|
-
if ! is_regular_file "$POLICY_PATH"; then
|
|
55
|
-
exit 0
|
|
56
|
-
fi
|
|
57
|
-
POLICY_CONTENT="$(<"$POLICY_PATH")" || exit 0
|
|
58
|
-
if [[ ${#POLICY_CONTENT} -gt 65536 ]]; then
|
|
59
|
-
exit 0
|
|
60
|
-
fi
|
|
61
|
-
MODE_OFF_PATTERN='"mode"[[:space:]]*:[[:space:]]*"off"'
|
|
62
|
-
MODE_ACTIVE_PATTERN='"mode"[[:space:]]*:[[:space:]]*"(observe|safe)"'
|
|
63
|
-
if [[ "$POLICY_CONTENT" =~ $MODE_OFF_PATTERN ]]; then
|
|
64
|
-
exit 0
|
|
65
|
-
fi
|
|
66
|
-
if [[ ! "$POLICY_CONTENT" =~ $MODE_ACTIVE_PATTERN ]]; then
|
|
67
|
-
exit 0
|
|
68
|
-
fi
|
|
69
|
-
|
|
70
|
-
RUNTIME_PATH="${AI_TOOLKIT_OUTPUT_FILTER_HOOK_RUNTIME:-${AI_TOOLKIT_OUTPUT_FILTER_CLI:-$HOME/.softspark/ai-toolkit/scripts/output_filter_hook.py}}"
|
|
71
|
-
if ! is_regular_file "$RUNTIME_PATH"; then
|
|
72
|
-
exit 0
|
|
73
|
-
fi
|
|
74
|
-
python3 -S "$RUNTIME_PATH" hook --policy "$POLICY_PATH" 2>/dev/null || true
|
|
75
|
-
|
|
76
|
-
exit 0
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"mode": "off",
|
|
3
|
-
"profiles": [
|
|
4
|
-
"repeat-lines",
|
|
5
|
-
"tap-success"
|
|
6
|
-
],
|
|
7
|
-
"maxInputBytes": 8388608,
|
|
8
|
-
"minSavingsBytes": 1024,
|
|
9
|
-
"minSavingsRatio": 0.15,
|
|
10
|
-
"recovery": {
|
|
11
|
-
"mode": "ephemeral",
|
|
12
|
-
"ttlMinutes": 60,
|
|
13
|
-
"maxSessionBytes": 33554432
|
|
14
|
-
}
|
|
15
|
-
}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
# Native output-filter benchmark corpus
|
|
2
|
-
|
|
3
|
-
The corpus is deterministic, synthetic, offline, and authored for ai-toolkit.
|
|
4
|
-
It measures pure profile transformation separately from hook process startup.
|
|
5
|
-
|
|
6
|
-
The gates are 20 ms p95 for inputs up to 100 KiB, 150 ms p95 for the 8 MiB
|
|
7
|
-
hard-cap case, at least 30% reduction, and peak traced allocation no greater
|
|
8
|
-
than three input sizes plus 16 MiB. The cold-process gate invokes the production
|
|
9
|
-
Bash wrapper with a fresh Python process for every sample in one native session;
|
|
10
|
-
its p95 limit is 75 ms. The default 100 samples keep the p95 gate stable enough
|
|
11
|
-
for release validation.
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"name": "repeat-lines-100k",
|
|
4
|
-
"profile": "repeat-lines",
|
|
5
|
-
"kind": "repeat",
|
|
6
|
-
"targetBytes": 102400,
|
|
7
|
-
"lineWidth": 96,
|
|
8
|
-
"maxP95Ms": 20
|
|
9
|
-
},
|
|
10
|
-
{
|
|
11
|
-
"name": "tap-success-2k",
|
|
12
|
-
"profile": "tap-success",
|
|
13
|
-
"kind": "tap",
|
|
14
|
-
"testCount": 2000,
|
|
15
|
-
"maxP95Ms": 20
|
|
16
|
-
},
|
|
17
|
-
{
|
|
18
|
-
"name": "repeat-lines-8m",
|
|
19
|
-
"profile": "repeat-lines",
|
|
20
|
-
"kind": "repeat",
|
|
21
|
-
"targetBytes": 8388608,
|
|
22
|
-
"lineWidth": 1024,
|
|
23
|
-
"maxP95Ms": 150
|
|
24
|
-
}
|
|
25
|
-
]
|