@allansantos-dev/smart-tool 0.9.11 → 0.9.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/setup_audit.py ADDED
@@ -0,0 +1,669 @@
1
+ #!/usr/bin/env python3
2
+ """Setup audit: what Claude Code and Codex actually load for a project, and where the pieces collide.
3
+
4
+ Collects the effective set of both harnesses (settings layers, enabled plugins, hooks, MCP servers, skills and the
5
+ instruction chain) and runs deterministic checks, each tied to the documented behavior it relies on. Nothing is
6
+ edited. What could not be checked is reported as not verified, never as ok. MCP servers are started (stdio) or
7
+ called (HTTP) the way a session would, with `initialize` and `tools/list`, unless probe=False. Secrets stay out of
8
+ the report: env and header values are never read into it, only their names.
9
+
10
+ Usage: python setup_audit.py [--project DIR] [--no-probe] [--json]
11
+ """
12
+ import argparse
13
+ import concurrent.futures
14
+ import hashlib
15
+ import json
16
+ import os
17
+ import queue
18
+ import re
19
+ import shutil
20
+ import socket
21
+ import subprocess
22
+ import threading
23
+ import time
24
+ import tomllib
25
+ import urllib.error
26
+ import urllib.parse
27
+ import urllib.request
28
+
29
+ import yaml
30
+
31
+ import atomic_io
32
+ import endpoint_sync
33
+ import paths
34
+
35
+ REPORT_DIR = os.path.join(paths.DATA_DIR, "setup-audit")
36
+ PROBE_TIMEOUT_S = 15
37
+ SKILL_DESCRIPTION_MAX = 1536
38
+ SKILL_BUDGET_CHARS = 8000
39
+ CODEX_PROJECT_DOC_MAX = 32 * 1024
40
+ CLAUDE_TOOLS = ("Bash", "Read", "Grep", "Glob", "LS", "Edit", "MultiEdit", "Write", "NotebookEdit", "WebFetch",
41
+ "WebSearch", "Task")
42
+ CODEX_TOOLS = ("Bash", "apply_patch")
43
+ DECIDING_EVENTS = ("PreToolUse",)
44
+ DOCS = {"claude_hooks": "https://code.claude.com/docs/en/hooks.md", "codex_hooks": "https://learn.chatgpt.com/docs/hooks.md",
45
+ "claude_skills": "https://code.claude.com/docs/en/skills.md",
46
+ "codex_skills": "https://learn.chatgpt.com/docs/build-skills.md",
47
+ "claude_memory": "https://code.claude.com/docs/en/memory.md",
48
+ "codex_agents_md": "https://developers.openai.com/codex/guides/agents-md",
49
+ "claude_mcp": "https://code.claude.com/docs/en/mcp.md", "mcp_lifecycle":
50
+ "https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle"}
51
+ _SCRIPT_RE = re.compile(r"""["']?([^\s"'|&;<>]+\.(?:js|cjs|mjs|py|ps1|sh|bat|cmd|exe))["']?""", re.IGNORECASE)
52
+ _SILENCED_RE = re.compile(r"\|\|\s*(?:true|exit\s+0)\b|2>\s*(?:/dev/null|nul)\b|;\s*exit\s+0\s*$", re.IGNORECASE)
53
+ _IMPORT_RE = re.compile(r"(?<![\w`])@((?:~|\.{1,2})?/?[\w.\-/\\]+\.\w+)")
54
+
55
+
56
+ def _home(*parts):
57
+ """Path under the user's home folder."""
58
+ return os.path.join(os.path.expanduser("~"), *parts)
59
+
60
+
61
+ def codex_dir():
62
+ """Codex's home: CODEX_HOME when set, else ~/.codex."""
63
+ return os.environ.get("CODEX_HOME") or _home(".codex")
64
+
65
+
66
+ def _managed_settings():
67
+ """Where an administrator's managed-settings.json can live on this OS (highest settings layer)."""
68
+ roots = (os.environ.get("ProgramFiles", r"C:\Program Files"), os.environ.get("ProgramData", r"C:\ProgramData"))
69
+ return [os.path.join(root, "ClaudeCode", "managed-settings.json") for root in roots] if os.name == "nt" else \
70
+ ["/Library/Application Support/ClaudeCode/managed-settings.json", "/etc/claude-code/managed-settings.json"]
71
+
72
+
73
+ class _Snapshot:
74
+ """Files read while collecting (path -> sha256) plus what could not be read."""
75
+
76
+ def __init__(self):
77
+ self.files, self.unverified = {}, []
78
+
79
+ def read(self, path):
80
+ try:
81
+ with open(path, "rb") as stream:
82
+ data = stream.read()
83
+ except FileNotFoundError:
84
+ return None
85
+ except OSError as exc:
86
+ self.unverified.append(f"{path}: {exc}")
87
+ return None
88
+ self.files[os.path.abspath(path)] = hashlib.sha256(data).hexdigest()
89
+ return data.decode("utf-8-sig", "replace")
90
+
91
+ def json(self, path):
92
+ text = self.read(path)
93
+ if text is None:
94
+ return None
95
+ try:
96
+ data = json.loads(text)
97
+ except ValueError as exc:
98
+ self.unverified.append(f"{path}: invalid JSON ({exc})")
99
+ return None
100
+ return data if isinstance(data, dict) else None
101
+
102
+ def toml(self, path):
103
+ text = self.read(path)
104
+ if text is None:
105
+ return None
106
+ try:
107
+ return tomllib.loads(text)
108
+ except tomllib.TOMLDecodeError as exc:
109
+ self.unverified.append(f"{path}: invalid TOML ({exc})")
110
+ return None
111
+
112
+
113
+ def _hooks_from(data, source, root=None):
114
+ """Hook entries of a settings/hooks.json object: one dict per hook with its event, matcher and target."""
115
+ found = []
116
+ for event, groups in ((data or {}).get("hooks") or {}).items():
117
+ for group in groups if isinstance(groups, list) else []:
118
+ for hook in (group.get("hooks") or []) if isinstance(group, dict) else []:
119
+ if isinstance(hook, dict):
120
+ found.append({"event": event, "matcher": group.get("matcher") or "", "type": hook.get("type"),
121
+ "command": hook.get("command"), "url": hook.get("url"),
122
+ "server": hook.get("server"), "tool": hook.get("tool"), "source": source,
123
+ "root": root})
124
+ return found
125
+
126
+
127
+ _VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
128
+
129
+
130
+ def _expand_vars(value, extra):
131
+ """${VAR} and ${VAR:-default} as Claude Code expands them in .mcp.json (extra: CLAUDE_PLUGIN_ROOT and such)."""
132
+ if not isinstance(value, str):
133
+ return value
134
+ return _VAR_RE.sub(lambda m: extra.get(m.group(1)) or os.environ.get(m.group(1)) or m.group(2) or m.group(0), value)
135
+
136
+
137
+ def _mcp_entries(servers, scope, source, prefix="", extra=None):
138
+ """MCP server entries of one config (enabled only); env and header values kept in private fields. prefix names
139
+ a plugin's servers the way Claude Code does (plugin:<plugin>:<server>)."""
140
+ entries, extra = [], extra or {}
141
+ for name, spec in (servers or {}).items():
142
+ if not isinstance(spec, dict) or spec.get("enabled") is False:
143
+ continue
144
+ spec = {k: ([_expand_vars(a, extra) for a in v] if isinstance(v, list) else
145
+ {kk: _expand_vars(vv, extra) for kk, vv in v.items()} if isinstance(v, dict) else
146
+ _expand_vars(v, extra)) for k, v in spec.items()}
147
+ entries.append({"name": prefix + name, "scope": scope, "source": source, "url": spec.get("url"),
148
+ "transport": "http" if spec.get("url") else "stdio", "command": spec.get("command"),
149
+ "args": [str(a) for a in spec.get("args") or []],
150
+ "env_names": sorted((spec.get("env") or {}).keys()),
151
+ "_env": spec.get("env") or {}, "_headers": dict(spec.get("headers") or spec.get("http_headers")
152
+ or {}),
153
+ "_bearer_env": spec.get("bearer_token_env_var"),
154
+ "disabled_tools": list(spec.get("disabled_tools") or [])})
155
+ return entries
156
+
157
+
158
+ def _frontmatter(text):
159
+ """YAML frontmatter of a SKILL.md as a dict ({} when absent or invalid)."""
160
+ match = re.match(r"^---\s*\n(.*?)\n---", text or "", re.S)
161
+ if not match:
162
+ return {}
163
+ try:
164
+ data = yaml.safe_load(match.group(1))
165
+ except yaml.YAMLError:
166
+ return {}
167
+ return data if isinstance(data, dict) else {}
168
+
169
+
170
+ def _skills(snapshot, folder, source, recursive=False, disabled=()):
171
+ """Skills (name, description, path) under folder, skipping names turned off in skillOverrides."""
172
+ found = []
173
+ pattern = "**/SKILL.md" if recursive else "*/SKILL.md"
174
+ from pathlib import Path
175
+ for path in sorted(Path(folder).glob(pattern)) if os.path.isdir(folder) else []:
176
+ if path.parent.name in disabled:
177
+ continue
178
+ meta = _frontmatter(snapshot.read(str(path)))
179
+ found.append({"name": str(meta.get("name") or path.parent.name), "path": str(path), "source": source,
180
+ "description": " ".join(str(meta.get("description") or "").split())})
181
+ return found
182
+
183
+
184
+ def _ancestors(project):
185
+ """Folders from the drive root down to project, in that order."""
186
+ chain, current = [], os.path.abspath(project)
187
+ while True:
188
+ chain.append(current)
189
+ parent = os.path.dirname(current)
190
+ if parent == current:
191
+ return list(reversed(chain))
192
+ current = parent
193
+
194
+
195
+ def collect_claude(project, snapshot):
196
+ """Claude Code's effective set for project: settings layers, enabled plugins, hooks, MCP, skills, CLAUDE.md."""
197
+ layers = [(path, "managed") for path in _managed_settings()] + [
198
+ (os.path.join(endpoint_sync.claude_dir(), "settings.json"), "user"),
199
+ (os.path.join(project, ".claude", "settings.json"), "project"),
200
+ (os.path.join(project, ".claude", "settings.local.json"), "local")]
201
+ hooks, enabled, overrides, settings = [], {}, {}, []
202
+ for path, scope in layers:
203
+ data = snapshot.json(path)
204
+ if data is None:
205
+ continue
206
+ settings.append({"scope": scope, "path": path, "disable_all_hooks": bool(data.get("disableAllHooks"))})
207
+ hooks += _hooks_from(data, f"settings:{scope}")
208
+ enabled.update({k: v for k, v in (data.get("enabledPlugins") or {}).items()})
209
+ overrides.update(data.get("skillOverrides") or {})
210
+ disabled_skills = {name for name, value in overrides.items() if value == "off"}
211
+ state = snapshot.json(endpoint_sync.claude_json()) or {}
212
+ mcp = _mcp_entries(state.get("mcpServers"), "user", endpoint_sync.claude_json())
213
+ project_state = (state.get("projects") or {}).get(os.path.abspath(project).replace("\\", "/")) or \
214
+ (state.get("projects") or {}).get(os.path.abspath(project)) or {}
215
+ mcp += _mcp_entries(project_state.get("mcpServers"), "local", endpoint_sync.claude_json())
216
+ project_mcp = snapshot.json(os.path.join(project, ".mcp.json")) or {}
217
+ mcp += _mcp_entries(project_mcp.get("mcpServers"), "project", os.path.join(project, ".mcp.json"))
218
+ skills = _skills(snapshot, os.path.join(endpoint_sync.claude_dir(), "skills"), "user", disabled=disabled_skills)
219
+ skills += _skills(snapshot, os.path.join(project, ".claude", "skills"), "project", disabled=disabled_skills)
220
+ plugins = []
221
+ installed = snapshot.json(os.path.join(endpoint_sync.claude_dir(), "plugins", "installed_plugins.json")) or {}
222
+ for plugin_id, installs in (installed.get("plugins") or {}).items():
223
+ if not enabled.get(plugin_id):
224
+ continue
225
+ for install in installs if isinstance(installs, list) else []:
226
+ root = install.get("installPath")
227
+ if install.get("projectPath") and os.path.abspath(install["projectPath"]) != os.path.abspath(project):
228
+ continue
229
+ if not root or not os.path.isdir(root):
230
+ snapshot.unverified.append(f"plugin {plugin_id}: install path missing ({root})")
231
+ continue
232
+ name = plugin_id.split("@")[0]
233
+ plugins.append({"id": plugin_id, "path": root})
234
+ hooks += _hooks_from(snapshot.json(os.path.join(root, "hooks", "hooks.json")), f"plugin:{name}", root)
235
+ plugin_mcp = snapshot.json(os.path.join(root, ".mcp.json")) or {}
236
+ mcp += _mcp_entries(plugin_mcp.get("mcpServers", plugin_mcp), f"plugin:{name}",
237
+ os.path.join(root, ".mcp.json"), f"plugin:{name}:", {"CLAUDE_PLUGIN_ROOT": root})
238
+ skills += [{**s, "name": f"{name}:{s['name']}"} for s in
239
+ _skills(snapshot, os.path.join(root, "skills"), f"plugin:{name}")]
240
+ instructions = []
241
+ for path, kind in [(os.path.join(endpoint_sync.claude_dir(), "CLAUDE.md"), "user")] + [
242
+ (os.path.join(folder, name), "project") for folder in _ancestors(project)
243
+ for name in ("CLAUDE.md", "CLAUDE.local.md", os.path.join(".claude", "CLAUDE.md"))]:
244
+ text = snapshot.read(path)
245
+ if text is not None:
246
+ instructions.append({"path": path, "kind": kind, "bytes": len(text.encode("utf-8")), "_text": text})
247
+ return {"settings": settings, "plugins": plugins, "hooks": hooks, "mcp": mcp, "skills": skills,
248
+ "instructions": instructions}
249
+
250
+
251
+ def collect_codex(project, snapshot):
252
+ """Codex's effective set for project: config.toml (MCP, hooks), hooks.json, skills, AGENTS.md chain."""
253
+ config = snapshot.toml(os.path.join(codex_dir(), "config.toml")) or {}
254
+ mcp = _mcp_entries(config.get("mcp_servers"), "user", os.path.join(codex_dir(), "config.toml"))
255
+ hooks = _hooks_from(snapshot.json(os.path.join(codex_dir(), "hooks.json")), "hooks.json")
256
+ hooks += _hooks_from(config, "config.toml")
257
+ skills = _skills(snapshot, os.path.join(codex_dir(), "skills"), "user", recursive=True)
258
+ skills += _skills(snapshot, _home(".agents", "skills"), "user-agents")
259
+ skills += _skills(snapshot, os.path.join(project, ".agents", "skills"), "project")
260
+ instructions = []
261
+ for name in ("AGENTS.override.md", "AGENTS.md"):
262
+ text = snapshot.read(os.path.join(codex_dir(), name))
263
+ if text is not None:
264
+ instructions.append({"path": os.path.join(codex_dir(), name), "kind": "user",
265
+ "bytes": len(text.encode("utf-8")), "_text": text})
266
+ break
267
+ git_root = next((folder for folder in reversed(_ancestors(project)) if os.path.exists(os.path.join(folder, ".git"))),
268
+ os.path.abspath(project))
269
+ for folder in _ancestors(project):
270
+ if len(folder) < len(git_root):
271
+ continue
272
+ for name in ("AGENTS.override.md", "AGENTS.md"):
273
+ text = snapshot.read(os.path.join(folder, name))
274
+ if text is not None:
275
+ instructions.append({"path": os.path.join(folder, name), "kind": "project",
276
+ "bytes": len(text.encode("utf-8")), "_text": text})
277
+ break
278
+ return {"hooks": hooks, "mcp": mcp, "skills": skills, "instructions": instructions,
279
+ "plugins_unverified": bool(config.get("plugins"))}
280
+
281
+
282
+ def _finding(check, severity, harness, subject, detail, source, evidence=()):
283
+ """One finding; its id is stable for the same check, harness and subject (used to diff runs)."""
284
+ key = hashlib.sha1(f"{check}\0{harness}\0{subject}".encode()).hexdigest()[:10]
285
+ return {"id": key, "check": check, "severity": severity, "harness": harness, "subject": subject,
286
+ "detail": detail, "source": source, "evidence": list(evidence)}
287
+
288
+
289
+ def _matches(matcher, tool):
290
+ """Whether a hook matcher (empty or * for all, else a regex over the tool name) covers tool."""
291
+ if matcher in ("", "*"):
292
+ return True
293
+ try:
294
+ return re.fullmatch(matcher, tool) is not None
295
+ except re.error:
296
+ return matcher == tool
297
+
298
+
299
+ def check_hook_overlap(harness, hooks, tools):
300
+ """Deciding hooks (PreToolUse) that run on the same tool: they run in parallel; a deny wins, and two rewrites
301
+ (updatedInput replaces the whole input) have no documented order."""
302
+ findings, groups = [], {}
303
+ for event in DECIDING_EVENTS:
304
+ for tool in tools:
305
+ matching = [h for h in hooks if h["event"] == event and _matches(h["matcher"], tool)]
306
+ if len(matching) > 1:
307
+ key = (event, tuple(sorted(f"{h['source']} {h['type']} {h['matcher'] or '*'}" for h in matching)))
308
+ groups.setdefault(key, []).append(tool)
309
+ doc = DOCS["claude_hooks"] if harness == "claude" else DOCS["codex_hooks"]
310
+ rule = ("all run in parallel; deny > defer > ask > allow, and two updatedInput rewrites have no documented order"
311
+ if harness == "claude" else "they run concurrently and any deny wins")
312
+ for (event, members), names in groups.items():
313
+ sources = sorted({m.split(" ")[0] for m in members})
314
+ findings.append(_finding("hook_overlap", "warn", harness, f"{event} on {', '.join(names)}",
315
+ f"{len(members)} hooks decide on the same call ({rule}): " + "; ".join(members)
316
+ + (". From different sources: check they do not rewrite the same input."
317
+ if len(sources) > 1 else "."), doc))
318
+ return findings
319
+
320
+
321
+ def _expand(token, root):
322
+ """A hook script path with ${CLAUDE_PLUGIN_ROOT}, ~ and environment variables expanded."""
323
+ token = token.replace("${CLAUDE_PLUGIN_ROOT}", root or "${CLAUDE_PLUGIN_ROOT}")
324
+ return os.path.expandvars(os.path.expanduser(token))
325
+
326
+
327
+ def check_hook_targets(harness, hooks, mcp_names, probe):
328
+ """Hooks that cannot work as configured: missing script, silenced errors, unknown MCP server, endpoint down."""
329
+ findings = []
330
+ doc = DOCS["claude_hooks"] if harness == "claude" else DOCS["codex_hooks"]
331
+ for hook in hooks:
332
+ where = f"{hook['source']} {hook['event']} {hook['matcher'] or '*'}"
333
+ if hook["type"] == "command" and hook["command"]:
334
+ for token in _SCRIPT_RE.findall(hook["command"]):
335
+ path = _expand(token, hook.get("root"))
336
+ if "$" in path or "%" in path:
337
+ continue
338
+ if not os.path.isabs(path):
339
+ path = os.path.join(hook["root"], path) if hook.get("root") else None
340
+ if path and not os.path.exists(path):
341
+ findings.append(_finding("hook_missing_script", "error", harness, where,
342
+ f"The hook runs {token}, which does not exist.", doc, [hook["command"]]))
343
+ if _SILENCED_RE.search(hook["command"]):
344
+ findings.append(_finding("hook_silenced", "warn", harness, where,
345
+ "The hook discards its errors (|| true / 2>/dev/null): when it breaks, "
346
+ "nothing shows.", doc, [hook["command"]]))
347
+ elif hook["type"] == "mcp_tool" and hook.get("server") and hook["server"] not in mcp_names:
348
+ findings.append(_finding("hook_unknown_server", "error", harness, where,
349
+ f"The hook calls MCP server '{hook['server']}', which is not configured here.",
350
+ doc))
351
+ elif hook["type"] == "http" and hook.get("url") and probe:
352
+ problem = _reachable(hook["url"])
353
+ if problem:
354
+ findings.append(_finding("hook_endpoint_down", "error", harness, where,
355
+ f"The hook posts to {hook['url']}, which does not answer: {problem}", doc))
356
+ return findings
357
+
358
+
359
+ def _reachable(url):
360
+ """None when a TCP connection to the URL's host and port opens, else the error."""
361
+ parsed = urllib.parse.urlsplit(url)
362
+ try:
363
+ with socket.create_connection((parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80)),
364
+ timeout=3):
365
+ return None
366
+ except OSError as exc:
367
+ return str(exc)
368
+
369
+
370
+ def check_skills(harness, skills):
371
+ """Skill name collisions, descriptions over Claude's per-skill cut, and the total over the list budget."""
372
+ findings, by_name = [], {}
373
+ doc = DOCS["claude_skills"] if harness == "claude" else DOCS["codex_skills"]
374
+ for skill in skills:
375
+ by_name.setdefault(skill["name"], []).append(skill)
376
+ for name, same in by_name.items():
377
+ if len(same) > 1:
378
+ findings.append(_finding("skill_name_collision", "warn", harness, name,
379
+ f"{len(same)} skills share the name '{name}': only one is offered.", doc,
380
+ [s["path"] for s in same]))
381
+ if harness == "claude":
382
+ for skill in skills:
383
+ if len(skill["description"]) > SKILL_DESCRIPTION_MAX:
384
+ findings.append(_finding("skill_description_long", "warn", harness, skill["name"],
385
+ f"Description of {len(skill['description'])} characters; Claude Code cuts "
386
+ f"each one at {SKILL_DESCRIPTION_MAX}.", doc, [skill["path"]]))
387
+ total = sum(len(s["name"]) + len(s["description"]) for s in skills)
388
+ if total > SKILL_BUDGET_CHARS:
389
+ findings.append(_finding("skill_budget", "warn", harness, f"{len(skills)} skills",
390
+ f"Names and descriptions add up to {total:,} characters, over the ~{SKILL_BUDGET_CHARS:,} "
391
+ "the skill list gets (Claude: 1% of a 200k context; Codex: 2% or 8,000 characters):"
392
+ " some skills are left out of what the model sees.", doc))
393
+ return findings
394
+
395
+
396
+ def check_instructions(project, claude, codex):
397
+ """AGENTS.md that only Codex reads, broken @imports in CLAUDE.md, and a Codex chain over its byte limit."""
398
+ findings = []
399
+ claude_texts = [i for i in claude["instructions"]]
400
+ agents_here = os.path.exists(os.path.join(project, "AGENTS.md"))
401
+ imports_agents = any(re.search(r"(?<![\w`])@\S*AGENTS\.md", i["_text"]) for i in claude_texts)
402
+ if agents_here and not imports_agents:
403
+ findings.append(_finding("agents_md_not_read_by_claude", "warn", "claude", os.path.join(project, "AGENTS.md"),
404
+ "Codex reads AGENTS.md; Claude Code reads only CLAUDE.md (and what it imports with "
405
+ "@AGENTS.md), so the two agents get different instructions here.", DOCS["claude_memory"]))
406
+ for item in claude_texts:
407
+ base = os.path.dirname(item["path"])
408
+ for target in _IMPORT_RE.findall(item["_text"]):
409
+ path = os.path.expanduser(target) if target.startswith("~") else os.path.join(base, target)
410
+ if not os.path.exists(path):
411
+ findings.append(_finding("broken_import", "error", "claude", f"{item['path']} @{target}",
412
+ "Imported file does not exist: that part of the instructions never loads.",
413
+ DOCS["claude_memory"], [item["path"]]))
414
+ total = sum(i["bytes"] for i in codex["instructions"] if i["kind"] == "project")
415
+ if total > CODEX_PROJECT_DOC_MAX:
416
+ findings.append(_finding("codex_agents_truncated", "error", "codex", project,
417
+ f"Project AGENTS.md chain is {total:,} bytes; Codex reads {CODEX_PROJECT_DOC_MAX:,} "
418
+ "(project_doc_max_bytes) and drops the rest.", DOCS["codex_agents_md"],
419
+ [i["path"] for i in codex["instructions"]]))
420
+ return findings
421
+
422
+
423
+ def _rpc_stdio(entry, cwd, timeout):
424
+ """initialize + tools/list over stdio; {"tools": [...]} or {"error", "kind": missing|down|timeout}."""
425
+ command = entry["command"]
426
+ resolved = command if os.path.isabs(command or "") else shutil.which(command or "")
427
+ if not resolved or not os.path.exists(resolved):
428
+ return {"error": f"command not found: {command}", "kind": "missing"}
429
+ env = {**os.environ, **{k: str(v) for k, v in entry["_env"].items()}}
430
+ flags = getattr(subprocess, "CREATE_NO_WINDOW", 0)
431
+ try:
432
+ proc = subprocess.Popen([resolved, *entry["args"]], cwd=cwd, env=env, stdin=subprocess.PIPE,
433
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8",
434
+ errors="replace", creationflags=flags)
435
+ except OSError as exc:
436
+ return {"error": f"did not start: {exc}", "kind": "down"}
437
+ lines, errors = queue.Queue(), []
438
+ threading.Thread(target=lambda: [lines.put(l) for l in proc.stdout] + [lines.put(None)], daemon=True).start()
439
+ threading.Thread(target=lambda: errors.extend(proc.stderr), daemon=True).start()
440
+ deadline = time.monotonic() + timeout
441
+
442
+ def call(request_id, method, params):
443
+ proc.stdin.write(json.dumps({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + "\n")
444
+ proc.stdin.flush()
445
+ while True:
446
+ try:
447
+ line = lines.get(timeout=max(0.1, deadline - time.monotonic()))
448
+ except queue.Empty:
449
+ raise TimeoutError(f"no answer to {method} in {timeout}s") from None
450
+ if line is None:
451
+ raise RuntimeError(f"exited: {''.join(errors).strip()[-200:] or 'no output'}")
452
+ try:
453
+ message = json.loads(line)
454
+ except ValueError:
455
+ continue
456
+ if message.get("id") == request_id:
457
+ if "error" in message:
458
+ raise RuntimeError(str(message["error"].get("message"))[:200])
459
+ return message["result"]
460
+ try:
461
+ call(1, "initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
462
+ "clientInfo": {"name": "smart-tool-setup-audit", "version": "1"}})
463
+ proc.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n")
464
+ return {"tools": call(2, "tools/list", {}).get("tools") or []}
465
+ except (OSError, RuntimeError, TimeoutError) as exc:
466
+ return {"error": str(exc), "kind": "timeout" if isinstance(exc, TimeoutError) else "down"}
467
+ finally:
468
+ if os.name == "nt":
469
+ subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, creationflags=flags)
470
+ else:
471
+ proc.kill()
472
+ try:
473
+ proc.wait(timeout=10)
474
+ except subprocess.TimeoutExpired:
475
+ pass
476
+ for stream in (proc.stdin, proc.stdout, proc.stderr):
477
+ try:
478
+ stream.close()
479
+ except OSError:
480
+ pass
481
+
482
+
483
+ def _rpc_http(entry, timeout):
484
+ """initialize + tools/list over streamable HTTP; {"tools": [...]} or {"error", "kind": auth|down}."""
485
+ headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream",
486
+ **{k: str(v) for k, v in entry["_headers"].items()}}
487
+ if entry.get("_bearer_env") and os.environ.get(entry["_bearer_env"]):
488
+ headers["Authorization"] = f"Bearer {os.environ[entry['_bearer_env']]}"
489
+
490
+ def post(body):
491
+ request = urllib.request.Request(entry["url"], data=json.dumps(body).encode(), headers=headers, method="POST")
492
+ with urllib.request.urlopen(request, timeout=timeout) as response:
493
+ if response.headers.get("Mcp-Session-Id"):
494
+ headers["Mcp-Session-Id"] = response.headers["Mcp-Session-Id"]
495
+ text = response.read().decode("utf-8", "replace")
496
+ if not text.strip():
497
+ return {}
498
+ if text.lstrip().startswith("{"):
499
+ return json.loads(text)
500
+ data = [line[5:].strip() for line in text.splitlines() if line.startswith("data:")]
501
+ return json.loads(data[-1]) if data else {}
502
+ try:
503
+ post({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {
504
+ "protocolVersion": "2025-06-18", "capabilities": {},
505
+ "clientInfo": {"name": "smart-tool-setup-audit", "version": "1"}}})
506
+ post({"jsonrpc": "2.0", "method": "notifications/initialized"})
507
+ answer = post({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}})
508
+ return {"tools": (answer.get("result") or {}).get("tools") or []}
509
+ except urllib.error.HTTPError as exc:
510
+ kind = "auth" if exc.code in (401, 403) else "down"
511
+ return {"error": f"HTTP {exc.code}", "kind": kind}
512
+ except (OSError, ValueError) as exc:
513
+ return {"error": str(getattr(exc, "reason", exc)), "kind": "down"}
514
+
515
+
516
+ def probe_mcp(entries, cwd, timeout=PROBE_TIMEOUT_S):
517
+ """Tools of each MCP server, started or called as a session would (in parallel)."""
518
+ def one(entry):
519
+ # A server whose config cannot even be used (a URL with an unexpanded variable, a bad port) is a finding.
520
+ try:
521
+ if entry["transport"] == "http":
522
+ return _rpc_http(entry, timeout)
523
+ return _rpc_stdio(entry, cwd, timeout)
524
+ except Exception as exc:
525
+ return {"error": f"{type(exc).__name__}: {exc}"[:200], "kind": "down"}
526
+ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
527
+ return list(pool.map(one, entries))
528
+
529
+
530
+ def check_mcp(harness, entries, results):
531
+ """Unhealthy servers, and tools offered twice (same name, or a proxy re-exposing a directly registered server)."""
532
+ findings, tools_by_server = [], {}
533
+ doc = DOCS["claude_mcp"] if harness == "claude" else DOCS["mcp_lifecycle"]
534
+ for entry, result in zip(entries, results):
535
+ subject = f"{entry['name']} ({entry['scope']})"
536
+ if result is None:
537
+ continue
538
+ if result.get("error"):
539
+ severity = {"missing": "error", "down": "error", "timeout": "warn", "auth": "info"}[result["kind"]]
540
+ text = {"auth": "needs sign-in; its tools were not verified",
541
+ "timeout": "did not answer in time; its tools were not verified"}.get(result["kind"],
542
+ "does not start or answer")
543
+ findings.append(_finding("mcp_unhealthy", severity, harness, subject,
544
+ f"MCP server {text}: {result['error']}", doc, [entry["source"]]))
545
+ continue
546
+ names = [t.get("name") for t in result["tools"] if t.get("name") not in entry["disabled_tools"]]
547
+ tools_by_server[entry["name"]] = set(names)
548
+ chars = sum(len(json.dumps(t)) for t in result["tools"])
549
+ entry["tools"], entry["tool_chars"] = len(names), chars
550
+ for server, names in tools_by_server.items():
551
+ for other, other_names in tools_by_server.items():
552
+ if other == server:
553
+ continue
554
+ same = sorted(n for n in names if n in other_names) if server < other else []
555
+ proxied = sorted(n for n in names if "__" in n and n.split("__", 1)[0] == other
556
+ and n.split("__", 1)[1] in other_names)
557
+ if same:
558
+ findings.append(_finding("mcp_duplicate_tools", "warn", harness, f"{server} and {other}",
559
+ f"{len(same)} tools with the same name on both servers: {', '.join(same[:6])}",
560
+ doc))
561
+ if proxied:
562
+ findings.append(_finding("mcp_duplicate_tools", "warn", harness, f"{server} proxies {other}",
563
+ f"{server} re-exposes {len(proxied)} tools of {other}, which is also "
564
+ f"registered directly: {', '.join(proxied[:6])}. The model sees both and "
565
+ "pays for both descriptions.", doc))
566
+ return findings
567
+
568
+
569
+ def _public(section):
570
+ """A collected section without private fields (instruction text, env and header values)."""
571
+ out = {}
572
+ for key, value in section.items():
573
+ if isinstance(value, list):
574
+ out[key] = [{k: v for k, v in item.items() if not k.startswith("_")} if isinstance(item, dict) else item
575
+ for item in value]
576
+ else:
577
+ out[key] = value
578
+ return out
579
+
580
+
581
+ def run(project, probe=True, save=True):
582
+ """Collects both harnesses for project, runs every check and returns the report; with save, stores it and
583
+ marks findings new or resolved against the previous report of the same project."""
584
+ project = os.path.abspath(project)
585
+ snapshot = _Snapshot()
586
+ claude, codex = collect_claude(project, snapshot), collect_codex(project, snapshot)
587
+ findings = []
588
+ for harness, data, tools in (("claude", claude, CLAUDE_TOOLS), ("codex", codex, CODEX_TOOLS)):
589
+ names = {m["name"] for m in data["mcp"]}
590
+ findings += check_hook_overlap(harness, data["hooks"], tools)
591
+ findings += check_hook_targets(harness, data["hooks"], names, probe)
592
+ findings += check_skills(harness, data["skills"])
593
+ results = probe_mcp(data["mcp"], project) if probe else [None] * len(data["mcp"])
594
+ findings += check_mcp(harness, data["mcp"], results)
595
+ findings += check_instructions(project, claude, codex)
596
+ unverified = list(snapshot.unverified)
597
+ if not probe:
598
+ unverified.append("MCP servers and HTTP hook endpoints were not contacted (probe off)")
599
+ if codex.get("plugins_unverified"):
600
+ unverified.append("Codex plugins (their skills and hooks) are not collected yet")
601
+ unverified += ["skill trigger overlap and contradictions between instructions need a model (not run)",
602
+ "the order between two parallel rewrites is undocumented and cannot be checked"]
603
+ key = hashlib.sha256(json.dumps(sorted(snapshot.files.items())).encode()).hexdigest()[:16]
604
+ order = {"error": 0, "warn": 1, "info": 2}
605
+ findings.sort(key=lambda f: (order[f["severity"]], f["harness"], f["check"], f["subject"]))
606
+ report = {"project": project, "checked_at": time.time(), "snapshot_key": key, "probe": probe,
607
+ "summary": {s: sum(f["severity"] == s for f in findings) for s in order},
608
+ "findings": findings, "unverified": unverified,
609
+ "claude": _public(claude), "codex": _public(codex), "files": len(snapshot.files)}
610
+ if save:
611
+ _save_with_diff(report)
612
+ return report
613
+
614
+
615
+ def _save_with_diff(report):
616
+ os.makedirs(REPORT_DIR, exist_ok=True)
617
+ last_path = os.path.join(REPORT_DIR, hashlib.sha1(report["project"].lower().encode()).hexdigest()[:12] + ".json")
618
+ try:
619
+ with open(last_path, encoding="utf-8") as stream:
620
+ previous = json.load(stream)
621
+ except (FileNotFoundError, ValueError):
622
+ previous = None
623
+ if previous:
624
+ before = {f["id"] for f in previous.get("findings", [])}
625
+ now = {f["id"] for f in report["findings"]}
626
+ report["changes"] = {"snapshot_changed": previous.get("snapshot_key") != report["snapshot_key"],
627
+ "new": sorted(now - before), "resolved": [f for f in previous["findings"]
628
+ if f["id"] not in now]}
629
+ for finding in report["findings"]:
630
+ finding["new"] = finding["id"] not in before
631
+ atomic_io.write_secret_text(last_path, json.dumps(report, indent=1, ensure_ascii=False))
632
+
633
+
634
+ def markdown(report):
635
+ """The report as Markdown for people: findings by severity, what changed, what was not verified."""
636
+ s = report["summary"]
637
+ lines = [f"# Setup audit: {report['project']}", "",
638
+ f"{s['error']} errors, {s['warn']} warnings, {s['info']} notes. Snapshot {report['snapshot_key']} "
639
+ f"({report['files']} files read).", ""]
640
+ changes = report.get("changes")
641
+ if changes:
642
+ lines += [f"Since the last run: {len(changes['new'])} new, {len(changes['resolved'])} resolved"
643
+ + (" (configuration changed)." if changes["snapshot_changed"] else "."), ""]
644
+ for finding in report["findings"]:
645
+ mark = {"error": "ERROR", "warn": "WARN", "info": "NOTE"}[finding["severity"]]
646
+ lines.append(f"- **{mark}** [{finding['harness']}] {finding['subject']}: {finding['detail']} "
647
+ f"([source]({finding['source']}))" + (" *(new)*" if finding.get("new") else ""))
648
+ for side in ("claude", "codex"):
649
+ servers = [m for m in report[side]["mcp"] if "tools" in m]
650
+ if servers:
651
+ lines += ["", f"MCP tools offered to {side}: " + ", ".join(
652
+ f"{m['name']} {m['tools']} (~{m['tool_chars'] // 4:,} tokens)" for m in servers)]
653
+ lines += ["", "Not verified:"] + [f"- {item}" for item in report["unverified"]]
654
+ return "\n".join(lines)
655
+
656
+
657
+ def main():
658
+ """CLI: prints the Markdown report (or JSON with --json)."""
659
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
660
+ parser.add_argument("--project", default=os.getcwd())
661
+ parser.add_argument("--no-probe", action="store_true")
662
+ parser.add_argument("--json", action="store_true")
663
+ args = parser.parse_args()
664
+ report = run(args.project, probe=not args.no_probe)
665
+ print(json.dumps(report, indent=1, ensure_ascii=False) if args.json else markdown(report))
666
+
667
+
668
+ if __name__ == "__main__":
669
+ main()