@allansantos-dev/smart-tool 0.9.79 → 0.9.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +132 -0
- package/README.md +82 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/client_hooks.py +10 -8
- package/code_health.py +182 -0
- package/code_impact.py +14 -2
- package/config.py +76 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +71 -9
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +289 -18
- package/hooks/pretool_router.py +5 -4
- package/indexer.py +92 -3
- package/install.py +66 -27
- package/install_runtime.py +24 -10
- package/layer_check.py +57 -0
- package/link_check.py +69 -0
- package/local_embedder.py +115 -58
- package/locales/pt-BR.json +122 -18
- package/low_priority.py +41 -0
- package/model_client.py +1 -1
- package/model_health.py +24 -6
- package/notice_action.py +9 -0
- package/package.json +1 -1
- package/pr_comment.py +94 -0
- package/pr_impact.py +292 -8
- package/process_limits.py +108 -0
- package/project_admission.py +17 -15
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/removed_callers.py +146 -0
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +241 -41
- package/smart_tool_daemon.py +403 -91
- package/tray.py +29 -9
- package/update_check.py +18 -0
- package/update_splash.py +2 -2
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_document_graph.py +12 -1
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/notice_action.py
CHANGED
|
@@ -5,6 +5,7 @@ Started by Windows through the protocol registered in `toast_notice.register`, w
|
|
|
5
5
|
refused) is appended to `notice-actions.log` in the data folder, because nothing else can show an error here.
|
|
6
6
|
"""
|
|
7
7
|
import os
|
|
8
|
+
import re
|
|
8
9
|
import sys
|
|
9
10
|
import time
|
|
10
11
|
import urllib.parse
|
|
@@ -18,6 +19,7 @@ import toast_notice
|
|
|
18
19
|
|
|
19
20
|
LOG_PATH = os.path.join(paths.DATA_DIR, "notice-actions.log")
|
|
20
21
|
LOG_MAX_BYTES = 256 * 1024
|
|
22
|
+
NPM_VERSION = re.compile(r"\d+\.\d+\.\d+(?:-[0-9A-Za-z][0-9A-Za-z.-]*)?")
|
|
21
23
|
|
|
22
24
|
|
|
23
25
|
def run(uri):
|
|
@@ -43,6 +45,13 @@ def run(uri):
|
|
|
43
45
|
import update_check
|
|
44
46
|
webbrowser.open(update_check.SITE_URL)
|
|
45
47
|
return "changes opened"
|
|
48
|
+
if action == "update-now":
|
|
49
|
+
import update_check
|
|
50
|
+
target = params.get("v", "")
|
|
51
|
+
if not NPM_VERSION.fullmatch(target):
|
|
52
|
+
raise ValueError(f"Not an npm version: {target[:40]!r}")
|
|
53
|
+
update_check.run_update(target, automatic=False)
|
|
54
|
+
return f"update to {target} started"
|
|
46
55
|
if action == "mute-model":
|
|
47
56
|
import model_health
|
|
48
57
|
return model_health.mute(params.get("model", ""))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.81",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Smart Tool contributors",
|
package/pr_comment.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Posts the `pr_impact` report as a comment on a pull request through the gh CLI.
|
|
2
|
+
|
|
3
|
+
This writes to GitHub with the user's own gh login, so it runs only while the setup screen's `pr_comment` consent is on.
|
|
4
|
+
The comment carries a hidden marker: a rerun edits the last comment that has it instead of stacking a new one.
|
|
5
|
+
"""
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import shutil
|
|
10
|
+
import subprocess
|
|
11
|
+
import tempfile
|
|
12
|
+
|
|
13
|
+
import config
|
|
14
|
+
import pr_impact
|
|
15
|
+
|
|
16
|
+
MARKER = "<!-- smart-tool:pr-impact -->"
|
|
17
|
+
GH_TIMEOUT_S = 60
|
|
18
|
+
COMMENT_ID = re.compile(r"#issuecomment-(\d+)$")
|
|
19
|
+
USER_FOLDER = re.compile(r"(?<![\w.\-/\\])(?:[A-Za-z]:)?[\\/](?:Users|home)[\\/][^\s`'\"|)\]]+")
|
|
20
|
+
DRIVE_PATH = re.compile(r"\b[A-Za-z]:[\\/][^\s`'\"|)\]]*")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _gh(gh, root, *args):
|
|
24
|
+
"""Runs gh from the project folder and returns its stdout; a failure says what gh answered."""
|
|
25
|
+
try:
|
|
26
|
+
done = subprocess.run([gh, *args], cwd=root, stdin=subprocess.DEVNULL, capture_output=True, text=True,
|
|
27
|
+
encoding="utf-8", errors="replace", timeout=GH_TIMEOUT_S,
|
|
28
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
29
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
30
|
+
raise RuntimeError(f"gh {' '.join(args[:2])} could not run: {exc}")
|
|
31
|
+
if done.returncode:
|
|
32
|
+
raise RuntimeError(f"gh {' '.join(args[:2])} failed: {(done.stderr or done.stdout).strip()[:300]}")
|
|
33
|
+
return done.stdout
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _relativize(text, root):
|
|
37
|
+
"""Drops the project folder from every path in text and replaces any other user-folder or drive path, so the
|
|
38
|
+
comment shows no location on the user's machine."""
|
|
39
|
+
absolute = os.path.abspath(root)
|
|
40
|
+
parts = [re.escape(part) for part in re.split(r"[\\/]+", absolute) if part]
|
|
41
|
+
folder = (r"[\\/]" if absolute[:1] in "/\\" else "") + r"[\\/]+".join(parts)
|
|
42
|
+
text = re.sub(folder + r"[\\/]+", "", text, flags=re.I)
|
|
43
|
+
text = re.sub(folder + r"(?![\w.\-])", ".", text, flags=re.I)
|
|
44
|
+
return DRIVE_PATH.sub("<local path>", USER_FOLDER.sub("<local path>", text))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _previous_comment_id(gh, root, pr):
|
|
48
|
+
"""The id of the last comment on the pull request that carries the marker, or None."""
|
|
49
|
+
comments = json.loads(_gh(gh, root, "pr", "view", str(pr), "--json", "comments"))["comments"]
|
|
50
|
+
for comment in reversed(comments):
|
|
51
|
+
found = COMMENT_ID.search(comment.get("url", ""))
|
|
52
|
+
if MARKER in comment.get("body", "") and found:
|
|
53
|
+
return found.group(1)
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def publish(root, pr, base, limit=30, view_id=None):
|
|
58
|
+
"""Posts the impact report of the change against base on pull request pr, or replaces the previous Smart Tool comment.
|
|
59
|
+
|
|
60
|
+
Parameters: root (str) project folder, where gh finds the repository; pr (int) pull request number; base (str) git
|
|
61
|
+
revision the report compares with; limit (int) rows per list; view_id (str or None).
|
|
62
|
+
Returns: dict with the comment 'url', whether it was 'updated' and the 'pr'.
|
|
63
|
+
Raises: RuntimeError when the consent is off, gh is missing or not logged in, or gh fails; ValueError for a bad pr or base.
|
|
64
|
+
"""
|
|
65
|
+
if config.pr_comment() != "on":
|
|
66
|
+
raise RuntimeError("pr_comment is off: nothing is posted to GitHub. The user turns it on in the Smart Tool setup "
|
|
67
|
+
"screen (Agent rules, Impact report as a pull request comment).")
|
|
68
|
+
if type(pr) is not int or pr < 1:
|
|
69
|
+
raise ValueError("pr must be the number of the pull request, a positive integer.")
|
|
70
|
+
if not isinstance(base, str) or not base:
|
|
71
|
+
raise ValueError("base is required: the git revision the pull request is compared with, such as origin/main.")
|
|
72
|
+
gh = shutil.which("gh")
|
|
73
|
+
if not gh:
|
|
74
|
+
raise RuntimeError("The gh CLI is not installed or not on PATH: install it from https://cli.github.com and run gh auth login.")
|
|
75
|
+
try:
|
|
76
|
+
_gh(gh, root, "auth", "status")
|
|
77
|
+
except RuntimeError as exc:
|
|
78
|
+
raise RuntimeError(f"gh is not logged in ({exc}); run gh auth login.")
|
|
79
|
+
body = _relativize(pr_impact.report(root, base, limit, view_id)["markdown"], root) + "\n" + MARKER + "\n"
|
|
80
|
+
previous = _previous_comment_id(gh, root, pr)
|
|
81
|
+
with tempfile.TemporaryDirectory() as folder:
|
|
82
|
+
path = os.path.join(folder, "comment.md")
|
|
83
|
+
with open(path, "w", encoding="utf-8", newline="\n") as stream:
|
|
84
|
+
stream.write(body)
|
|
85
|
+
if previous:
|
|
86
|
+
answer = json.loads(_gh(gh, root, "api", "--method", "PATCH",
|
|
87
|
+
f"repos/{{owner}}/{{repo}}/issues/comments/{previous}", "-F", f"body=@{path}"))
|
|
88
|
+
url = answer.get("html_url", "")
|
|
89
|
+
else:
|
|
90
|
+
lines = _gh(gh, root, "pr", "comment", str(pr), "--body-file", path).strip().splitlines()
|
|
91
|
+
url = lines[-1] if lines else ""
|
|
92
|
+
if not url.startswith("https://"):
|
|
93
|
+
raise RuntimeError("gh did not return the comment URL, so the post is not confirmed; look at the pull request.")
|
|
94
|
+
return {"url": url, "updated": bool(previous), "pr": pr}
|
package/pr_impact.py
CHANGED
|
@@ -1,26 +1,274 @@
|
|
|
1
1
|
"""Impact report of a change set, in Markdown, for the description or a comment of a pull request.
|
|
2
2
|
|
|
3
3
|
It joins what Smart Tool already computes: the functions the diff touches and the tests to run (affected_tests), the
|
|
4
|
-
callers of each touched function (code_impact)
|
|
4
|
+
callers of each touched function and the tests that reach it (code_impact), the documentation anchors the change leaves
|
|
5
|
+
out of date (doc_scope), the files that usually change with the diff's files and are missing from it, the import cycles
|
|
6
|
+
through the changed files and the changed files that are hotspots (code_health), the new functions no caller or test reaches (code_health), the callers that still name a function
|
|
7
|
+
the diff removed or renamed and the callers of those whose signature changed (removed_callers), the decision notes that name a touched function (dossier) and the notes the
|
|
8
|
+
edit hook would have added to the whole diff (hook_decision).
|
|
5
9
|
Nothing is published: the agent pastes the report where it belongs.
|
|
6
10
|
"""
|
|
7
11
|
import os
|
|
8
12
|
|
|
9
13
|
import affected_tests
|
|
14
|
+
import code_health
|
|
10
15
|
import code_impact
|
|
16
|
+
import config
|
|
17
|
+
import doc_check
|
|
11
18
|
import doc_scope
|
|
19
|
+
import dossier
|
|
20
|
+
import hook_decision
|
|
21
|
+
import removed_callers
|
|
12
22
|
|
|
13
23
|
MAX_FUNCTIONS = 15
|
|
14
24
|
MAX_CALLERS = 8
|
|
25
|
+
TEST_DEPTH = 3
|
|
26
|
+
MAX_CHECKED_FILES = 50
|
|
27
|
+
MAX_COUPLED = 10
|
|
28
|
+
MAX_HOTSPOTS = 10
|
|
29
|
+
ALL_ROWS = 100000
|
|
15
30
|
|
|
16
31
|
|
|
17
32
|
def _callers(root, label, view_id):
|
|
18
|
-
"""Callers of one touched function, or None when the function is new and nothing calls it yet."""
|
|
33
|
+
"""Callers and reaching tests of one touched function, or None when the function is new and nothing calls it yet."""
|
|
19
34
|
try:
|
|
20
|
-
found = code_impact.impact(root, label, view_id, depth=
|
|
35
|
+
found = code_impact.impact(root, label, view_id, depth=TEST_DEPTH, limit=MAX_CALLERS)
|
|
21
36
|
except ValueError:
|
|
22
37
|
return None
|
|
23
|
-
return found["callers"], found["counts"]["callers"]
|
|
38
|
+
return found["callers"], found["counts"]["callers"], found["counts"]["tests"]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _versions(root, base, changes):
|
|
42
|
+
"""The text of each changed file at the base and now.
|
|
43
|
+
|
|
44
|
+
Parameters: root (str) project folder; base (str) git revision; changes (dict) the `affected_tests.affected` answer.
|
|
45
|
+
Returns: list of (project-relative path, absolute path, text at the base or None when the file is new, text now or None
|
|
46
|
+
when the file is gone); files that cannot be read as text are left out.
|
|
47
|
+
"""
|
|
48
|
+
paths = sorted({*changes["changed"]["code"], *changes["changed"]["tests"], *changes["changed"]["other"]})
|
|
49
|
+
found = []
|
|
50
|
+
for rel in paths[:MAX_CHECKED_FILES]:
|
|
51
|
+
full = os.path.join(root, *rel.split("/"))
|
|
52
|
+
after = None
|
|
53
|
+
if os.path.isfile(full):
|
|
54
|
+
try:
|
|
55
|
+
with open(full, encoding="utf-8") as stream:
|
|
56
|
+
after = stream.read()
|
|
57
|
+
except (OSError, UnicodeDecodeError):
|
|
58
|
+
continue
|
|
59
|
+
try:
|
|
60
|
+
before = affected_tests._git(root, "show", f"{base}:./{rel}")
|
|
61
|
+
except ValueError:
|
|
62
|
+
before = None
|
|
63
|
+
found.append((rel, full, before, after))
|
|
64
|
+
return found
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _hook_notes(versions):
|
|
68
|
+
"""The edit hook's notes over the changed files of the diff, which commits made without the hook never got.
|
|
69
|
+
|
|
70
|
+
Parameters: versions (list) `_versions` of the diff.
|
|
71
|
+
Returns: tuple (str notes, "" when nothing is worth saying; str problem, "" when the checks ran).
|
|
72
|
+
"""
|
|
73
|
+
triples = [(full, before, after) for _rel, full, before, after in versions if after is not None]
|
|
74
|
+
try:
|
|
75
|
+
return hook_decision.review_diff(triples, config.load_config()), ""
|
|
76
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
77
|
+
return "", f"the edit checks could not run: {exc}"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _reviewed(root, versions, changed_code, review, key, what):
|
|
81
|
+
"""The answers of `review` over the changed code files that have something under `key`, or the reason it could not run.
|
|
82
|
+
|
|
83
|
+
Returns: tuple (list of answers; str problem, "" when the check ran).
|
|
84
|
+
"""
|
|
85
|
+
answers = []
|
|
86
|
+
try:
|
|
87
|
+
for rel, _full, before, after in versions:
|
|
88
|
+
if rel in changed_code:
|
|
89
|
+
answer = review(root, rel, before, after or "")
|
|
90
|
+
if answer[key]:
|
|
91
|
+
answers.append(answer)
|
|
92
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
93
|
+
return answers, f"the callers of {what} could not be read: {exc}"
|
|
94
|
+
return answers, ""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _removed_callers(root, versions, changed_code):
|
|
98
|
+
"""The callers that still name a function the diff removed or renamed, in the changed code files.
|
|
99
|
+
|
|
100
|
+
Returns: tuple (list of answers of `removed_callers.review`, one per file that lost a function; str problem, "" when the
|
|
101
|
+
check ran).
|
|
102
|
+
"""
|
|
103
|
+
return _reviewed(root, versions, changed_code, removed_callers.review, "removed", "removed functions")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _signature_callers(root, versions, changed_code):
|
|
107
|
+
"""The callers outside the file of the functions whose signature the diff changed, in the changed code files.
|
|
108
|
+
|
|
109
|
+
Returns: tuple (list of answers of `removed_callers.signature_review`, one per file with a changed signature; str problem,
|
|
110
|
+
"" when the check ran).
|
|
111
|
+
"""
|
|
112
|
+
return _reviewed(root, versions, changed_code, removed_callers.signature_review, "changed", "functions with a changed signature")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _also_defined(item):
|
|
116
|
+
return f" A function named {item['name'].rpartition('.')[2]} also exists in {', '.join(item['also_defined_in'])}." \
|
|
117
|
+
if item["also_defined_in"] else ""
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _callers_section(answers, problem, title, key, finding, quiet_text):
|
|
121
|
+
"""The Markdown lines of one caller section, none when there is nothing to say.
|
|
122
|
+
|
|
123
|
+
Parameters: answers (list) of `removed_callers` answers; problem (str) why the check could not run, "" when it ran;
|
|
124
|
+
title (str) the `###` heading; key (str) the answer key holding the names of the functions concerned; finding (str)
|
|
125
|
+
format of a caller line over `item` and `sites`; quiet_text (str) format of the line for names with no caller.
|
|
126
|
+
"""
|
|
127
|
+
if not answers and not problem:
|
|
128
|
+
return []
|
|
129
|
+
lines = [f"### {title}", ""]
|
|
130
|
+
for answer in answers:
|
|
131
|
+
for item in answer["findings"]:
|
|
132
|
+
lines.append(finding.format(item=item, sites=removed_callers.sites_text(item), also=_also_defined(item)))
|
|
133
|
+
quiet = [name for name in answer[key] if name not in {item["name"] for item in answer["findings"]}]
|
|
134
|
+
if quiet:
|
|
135
|
+
lines.append(quiet_text.format(names=", ".join(f"`{name}`" for name in quiet)))
|
|
136
|
+
lines += list(dict.fromkeys(answer["note"] for answer in answers if answer["note"]))
|
|
137
|
+
return lines + ([problem] if problem else []) + [""]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _removed_lines(answers, problem):
|
|
141
|
+
"""The Markdown lines of the section on removed or renamed functions, none when the diff removes no function."""
|
|
142
|
+
return _callers_section(answers, problem, "Callers of removed or renamed functions", "removed",
|
|
143
|
+
"- `{item[name]}` ({item[path]}) is gone and is still called from {sites}.{also}",
|
|
144
|
+
"No caller was found for the removed {names}.")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _signature_lines(answers, problem):
|
|
148
|
+
"""The Markdown lines of the section on functions whose signature changed, none when the diff changes no signature."""
|
|
149
|
+
return _callers_section(answers, problem, "Callers of functions whose signature changed", "changed",
|
|
150
|
+
"- `{item[name]}` ({item[path]}) has a new signature and is called from {sites} outside the file.{also}",
|
|
151
|
+
"No caller outside the file was found for {names}, whose signature changed.")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _coupled(root, view_id, changes):
|
|
155
|
+
"""Files that usually change with the diff's files but are not in it, or the reason none could be computed."""
|
|
156
|
+
paths = sorted({*changes["changed"]["code"], *changes["changed"]["tests"], *changes["changed"]["other"]})
|
|
157
|
+
try:
|
|
158
|
+
found = code_health.coupled_with(root, paths, view_id, MAX_COUPLED)
|
|
159
|
+
except ValueError as exc:
|
|
160
|
+
return [], f"The change coupling was not read: {exc}"
|
|
161
|
+
return found["files"], f"The change coupling was not read: {found['note']}" if found["note"] else None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _cycles(root, view_id, changed_code):
|
|
165
|
+
"""Import cycles that contain a changed code file, or the reason none could be computed.
|
|
166
|
+
|
|
167
|
+
Returns: tuple (list of `code_health.import_cycles` entries; str note, what the graph cut; str problem, "" when the
|
|
168
|
+
cycles were read).
|
|
169
|
+
"""
|
|
170
|
+
try:
|
|
171
|
+
found = code_health.import_cycles(root, view_id, limit=ALL_ROWS)
|
|
172
|
+
except (ValueError, RuntimeError) as exc:
|
|
173
|
+
return [], "", f"The import cycles were not read: {exc}"
|
|
174
|
+
return [c for c in found["cycles"] if changed_code.intersection(c["files"])], found["note"], ""
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _hotspots(root, view_id, changed_code):
|
|
178
|
+
"""Changed code files among the hotspots of the project, or the reason none could be computed.
|
|
179
|
+
|
|
180
|
+
Returns: tuple (list of `code_health.hotspots` rows, riskiest first; str problem, "" when the history was read).
|
|
181
|
+
"""
|
|
182
|
+
try:
|
|
183
|
+
found = code_health.hotspots(root, view_id, limit=ALL_ROWS)
|
|
184
|
+
except (ValueError, RuntimeError) as exc:
|
|
185
|
+
return [], f"The hotspots were not read: {exc}"
|
|
186
|
+
if found["note"]:
|
|
187
|
+
return [], f"The hotspots were not read: {found['note']}"
|
|
188
|
+
return [f for f in found["files"] if f["path"] in changed_code][:MAX_HOTSPOTS], ""
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _added_functions(versions, changed_code):
|
|
192
|
+
"""The (path, qualified name) pairs of the functions the diff adds: those a changed code file defines now and did not at
|
|
193
|
+
the base, every function of a new file included."""
|
|
194
|
+
added = set()
|
|
195
|
+
for rel, _full, before, after in versions:
|
|
196
|
+
if rel in changed_code and after is not None and doc_check.supported(rel):
|
|
197
|
+
known = {f["name"] for f in doc_check.functions(rel, before)} if before is not None else set()
|
|
198
|
+
added.update((rel, f["name"]) for f in doc_check.functions(rel, after) if f["name"] not in known)
|
|
199
|
+
return added
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _new_dead(root, view_id, versions, changed_code):
|
|
203
|
+
"""The functions the diff adds that no function and no test calls, from `code_health.dead_code`.
|
|
204
|
+
|
|
205
|
+
Returns: tuple (list of `dead_code` entries; str note, what the graph cut; str problem, "" when the check ran). Nothing is
|
|
206
|
+
asked of the graph when the diff adds no function.
|
|
207
|
+
"""
|
|
208
|
+
added = _added_functions(versions, changed_code)
|
|
209
|
+
if not added:
|
|
210
|
+
return [], "", ""
|
|
211
|
+
try:
|
|
212
|
+
found = code_health.dead_code(root, view_id, limit=ALL_ROWS)
|
|
213
|
+
except (ValueError, RuntimeError) as exc:
|
|
214
|
+
return [], "", f"The new functions without a caller were not read: {exc}"
|
|
215
|
+
return [f for f in found["no_caller"] if (f["path"], f["name"]) in added], found["graph_note"], ""
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _new_dead_lines(dead, note, problem):
|
|
219
|
+
"""The Markdown section on new functions nothing calls; left out when the diff adds none or all have a caller, unless
|
|
220
|
+
the graph was cut or the check could not run, which are said."""
|
|
221
|
+
if not (dead or note or problem):
|
|
222
|
+
return []
|
|
223
|
+
lines = ["### New functions nothing calls", ""]
|
|
224
|
+
lines += [f"- `{f['name']}` ({f['path']}:{f['line']})" + ("" if f["public"] else " - private") for f in dead]
|
|
225
|
+
if dead:
|
|
226
|
+
lines.append("Candidates from the static call graph (Python and Go only): a call through a framework, a decorator or a "
|
|
227
|
+
"name built at run time is not seen.")
|
|
228
|
+
elif not problem:
|
|
229
|
+
lines.append(f"No new function without a caller was found. {note}".rstrip())
|
|
230
|
+
note = ""
|
|
231
|
+
lines += [note] if note else []
|
|
232
|
+
return lines + ([problem] if problem else []) + [""]
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _shape_lines(cycles, cycles_note, cycles_problem, hotspots, hotspots_problem, changed_code):
|
|
236
|
+
"""The Markdown sections on import cycles and hotspots of the changed files; a section with nothing to say is left out
|
|
237
|
+
unless the graph was cut or the check could not run, which are said."""
|
|
238
|
+
lines = []
|
|
239
|
+
if cycles or cycles_note or cycles_problem:
|
|
240
|
+
lines += ["### Import cycles through the changed files", ""]
|
|
241
|
+
lines += [f"- {c['size']} files import each other, directly or through a chain: "
|
|
242
|
+
+ ", ".join(f"`{f}`" for f in c["files"]) + "; changed: "
|
|
243
|
+
+ ", ".join(f"`{f}`" for f in c["files"] if f in changed_code) for c in cycles]
|
|
244
|
+
if not cycles and not cycles_problem:
|
|
245
|
+
lines.append(f"No cycle through the changed files was found. {cycles_note}")
|
|
246
|
+
lines += [cycles_problem] if cycles_problem else []
|
|
247
|
+
lines.append("")
|
|
248
|
+
if hotspots or hotspots_problem:
|
|
249
|
+
lines += ["### Changed files that are hotspots", ""]
|
|
250
|
+
lines += [f"- `{f['path']}`: {f['commits']} commits, {f['lines']} lines, last changed {f['last_change']}" for f in hotspots]
|
|
251
|
+
lines += [hotspots_problem] if hotspots_problem else []
|
|
252
|
+
lines.append("")
|
|
253
|
+
return lines
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _decisions(root, labels):
|
|
257
|
+
"""Decision notes and commit messages that name each touched function (`dossier.notes_about`, whose rule keeps a
|
|
258
|
+
single word out of ordinary prose).
|
|
259
|
+
|
|
260
|
+
Returns: tuple (dict label -> list of hits, str problem, "" when the search ran whole).
|
|
261
|
+
"""
|
|
262
|
+
found, reasons = {}, []
|
|
263
|
+
for label in labels:
|
|
264
|
+
rel, _, qualified = label.partition("::")
|
|
265
|
+
if not qualified:
|
|
266
|
+
continue
|
|
267
|
+
answer = dossier.notes_about(root, rel, qualified)
|
|
268
|
+
reasons += [reason for reason in answer["reasons"] if reason not in reasons]
|
|
269
|
+
if answer["hits"]:
|
|
270
|
+
found[label] = answer["hits"]
|
|
271
|
+
return found, f"The notes search was cut: {'; '.join(reasons)}" if reasons else ""
|
|
24
272
|
|
|
25
273
|
|
|
26
274
|
def _stale(root, view_id, paths):
|
|
@@ -42,23 +290,39 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
42
290
|
touched = changes["touched_functions"][:MAX_FUNCTIONS]
|
|
43
291
|
callers = {label: _callers(root, label, view_id) for label in touched}
|
|
44
292
|
stale, stale_note = _stale(root, view_id, changed_code)
|
|
293
|
+
untested = [label for label in touched if callers[label] is None or callers[label][2] == 0]
|
|
294
|
+
versions = _versions(root, base, changes)
|
|
295
|
+
notes, notes_problem = _hook_notes(versions)
|
|
296
|
+
removed, removed_problem = _removed_callers(root, versions, changed_code)
|
|
297
|
+
resigned, resigned_problem = _signature_callers(root, versions, changed_code)
|
|
298
|
+
coupled, coupled_note = _coupled(root, view_id, changes)
|
|
299
|
+
decisions, decisions_note = _decisions(root, touched)
|
|
300
|
+
cycles, cycles_note, cycles_problem = _cycles(root, view_id, changed_code) if changed_code else ([], "", "")
|
|
301
|
+
hotspots, hotspots_problem = _hotspots(root, view_id, changed_code) if changed_code else ([], "")
|
|
302
|
+
new_dead, new_dead_note, new_dead_problem = _new_dead(root, view_id, versions, changed_code)
|
|
45
303
|
lines = [f"## Impact of the change against `{base}`", "",
|
|
46
304
|
f"{changes['counts']['changed_files']} changed file(s), {len(changes['touched_functions'])} touched "
|
|
47
305
|
f"function(s), {changes['counts']['tests']} test file(s) to run.", ""]
|
|
48
306
|
if changes["touched_functions"]:
|
|
49
|
-
lines += ["### Touched functions and who calls them", "",
|
|
307
|
+
lines += ["### Touched functions and who calls them", "",
|
|
308
|
+
"| Function | Callers | Tests | Called from |", "|---|---|---|---|"]
|
|
50
309
|
for label in touched:
|
|
51
310
|
reached = callers[label]
|
|
52
311
|
if reached is None:
|
|
53
|
-
lines.append(f"| `{label}` | 0 | new: nothing calls it yet |")
|
|
312
|
+
lines.append(f"| `{label}` | 0 | none | new: nothing calls it yet |")
|
|
54
313
|
continue
|
|
55
|
-
sites, total = reached
|
|
314
|
+
sites, total, tests = reached
|
|
56
315
|
shown = ", ".join(f"`{s['function']}` ({s['path']}:{s['line']})" for s in sites)
|
|
57
316
|
more = f" (+{total - len(sites)} more)" if total > len(sites) else ""
|
|
58
|
-
lines.append(f"| `{label}` | {total} | {shown or '-'}{more} |")
|
|
317
|
+
lines.append(f"| `{label}` | {total} | {tests or 'none'} | {shown or '-'}{more} |")
|
|
59
318
|
if len(changes["touched_functions"]) > len(touched):
|
|
60
319
|
lines.append(f"\n{len(changes['touched_functions']) - len(touched)} more touched function(s) not listed.")
|
|
61
320
|
lines.append("")
|
|
321
|
+
lines += ["### Touched functions no test reaches", ""]
|
|
322
|
+
lines += [f"- `{label}`" for label in untested] if untested else ["Every touched function is reached by a test."]
|
|
323
|
+
lines.append("")
|
|
324
|
+
lines += _removed_lines(removed, removed_problem)
|
|
325
|
+
lines += _signature_lines(resigned, resigned_problem)
|
|
62
326
|
lines += ["### Documentation left out of date", ""]
|
|
63
327
|
if stale_note:
|
|
64
328
|
lines += [stale_note, ""]
|
|
@@ -67,6 +331,23 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
67
331
|
lines += ["", "Regenerate with `project_manage action=docs generate=true` and review the prose of these entries.", ""]
|
|
68
332
|
else:
|
|
69
333
|
lines += ["No documented entry in the changed files is out of date.", ""]
|
|
334
|
+
lines += ["### Edit checks over the whole diff", "", notes or notes_problem or "The edit checks find nothing to add.", ""]
|
|
335
|
+
if touched:
|
|
336
|
+
lines += ["### Decision notes of the touched functions", ""]
|
|
337
|
+
lines += [f"- `{label}`: `{hit['id']}` {hit['title']}" for label, hits in decisions.items() for hit in hits]
|
|
338
|
+
if not decisions:
|
|
339
|
+
lines.append("No decision note or commit message names a touched function.")
|
|
340
|
+
lines += ([decisions_note] if decisions_note else []) + [""]
|
|
341
|
+
lines += ["### Files that usually change with these", ""]
|
|
342
|
+
if coupled_note:
|
|
343
|
+
lines += [coupled_note, ""]
|
|
344
|
+
elif coupled:
|
|
345
|
+
lines += [f"- `{f['path']}` changed in {f['shared']} commits with `{f['with']}` ({f['degree']}%) and is not in this diff"
|
|
346
|
+
for f in coupled] + [""]
|
|
347
|
+
else:
|
|
348
|
+
lines += ["No file that usually changes with these is missing from the diff.", ""]
|
|
349
|
+
lines += _shape_lines(cycles, cycles_note, cycles_problem, hotspots, hotspots_problem, changed_code)
|
|
350
|
+
lines += _new_dead_lines(new_dead, new_dead_note, new_dead_problem)
|
|
70
351
|
lines += ["### Tests to run", ""]
|
|
71
352
|
if changes["tests"]:
|
|
72
353
|
lines += [f"- `{t['path']}`: {t['why']}" for t in changes["tests"]]
|
|
@@ -82,5 +363,8 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
82
363
|
lines += ["", changes["note"]]
|
|
83
364
|
return {"base": base, "markdown": "\n".join(lines).rstrip("\n") + "\n", "touched": touched,
|
|
84
365
|
"callers": {label: (reached[1] if reached else 0) for label, reached in callers.items()},
|
|
366
|
+
"untested": untested, "coupled": coupled, "cycles": cycles, "hotspots": hotspots, "new_without_caller": new_dead, "decisions": decisions, "edit_checks": notes or notes_problem,
|
|
367
|
+
"removed_callers": [item for answer in removed for item in answer["findings"]],
|
|
368
|
+
"signature_callers": [item for answer in resigned for item in answer["findings"]],
|
|
85
369
|
"stale_anchors": stale, "stale_note": stale_note, "tests": changes["tests"],
|
|
86
370
|
"commands": changes["commands"], "run_all": changes["run_all"], "counts": changes["counts"]}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Child processes under a time cap and a memory cap whose whole tree dies with them."""
|
|
2
|
+
import os
|
|
3
|
+
import subprocess
|
|
4
|
+
|
|
5
|
+
_IS_WINDOWS = os.name == "nt"
|
|
6
|
+
JOB_OBJECT_LIMIT_JOB_MEMORY = 0x200
|
|
7
|
+
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
|
|
8
|
+
STDERR_TAIL_BYTES = 2000
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def kill_on_close_job(job_memory_bytes=None):
|
|
12
|
+
"""A Windows job object that kills every process in it when its handle closes, and, with `job_memory_bytes`, caps
|
|
13
|
+
the memory the whole tree may commit.
|
|
14
|
+
|
|
15
|
+
Parameters: job_memory_bytes (cap for all processes of the job together, None for no cap). Returns: `(kernel32,
|
|
16
|
+
job)`, the library and the job handle. Raises: OSError when the system refuses the job."""
|
|
17
|
+
import ctypes
|
|
18
|
+
from ctypes import wintypes
|
|
19
|
+
|
|
20
|
+
class IoCounters(ctypes.Structure):
|
|
21
|
+
_fields_ = [(name, ctypes.c_ulonglong) for name in
|
|
22
|
+
("read_ops", "write_ops", "other_ops", "read_bytes", "write_bytes", "other_bytes")]
|
|
23
|
+
|
|
24
|
+
class BasicLimits(ctypes.Structure):
|
|
25
|
+
_fields_ = [("per_process_time", ctypes.c_longlong), ("per_job_time", ctypes.c_longlong),
|
|
26
|
+
("flags", wintypes.DWORD), ("min_ws", ctypes.c_size_t), ("max_ws", ctypes.c_size_t),
|
|
27
|
+
("active_processes", wintypes.DWORD), ("affinity", ctypes.c_size_t),
|
|
28
|
+
("priority", wintypes.DWORD), ("scheduling", wintypes.DWORD)]
|
|
29
|
+
|
|
30
|
+
class ExtendedLimits(ctypes.Structure):
|
|
31
|
+
_fields_ = [("basic", BasicLimits), ("io", IoCounters), ("process_memory", ctypes.c_size_t),
|
|
32
|
+
("job_memory", ctypes.c_size_t), ("peak_process_memory", ctypes.c_size_t),
|
|
33
|
+
("peak_job_memory", ctypes.c_size_t)]
|
|
34
|
+
|
|
35
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
36
|
+
kernel32.CreateJobObjectW.restype = wintypes.HANDLE
|
|
37
|
+
job = kernel32.CreateJobObjectW(None, None)
|
|
38
|
+
if not job:
|
|
39
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
40
|
+
limits = ExtendedLimits()
|
|
41
|
+
limits.basic.flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
42
|
+
if job_memory_bytes:
|
|
43
|
+
limits.basic.flags |= JOB_OBJECT_LIMIT_JOB_MEMORY
|
|
44
|
+
limits.job_memory = job_memory_bytes
|
|
45
|
+
if not kernel32.SetInformationJobObject(wintypes.HANDLE(job), 9, ctypes.byref(limits), ctypes.sizeof(limits)):
|
|
46
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
47
|
+
return kernel32, job
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _stderr_tail(path):
|
|
51
|
+
with open(path, "rb") as stream:
|
|
52
|
+
stream.seek(0, os.SEEK_END)
|
|
53
|
+
stream.seek(max(0, stream.tell() - STDERR_TAIL_BYTES))
|
|
54
|
+
return stream.read().decode("utf-8", "replace").strip()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def run_capped(argv, workdir, timeout, memory_bytes):
|
|
58
|
+
"""Runs `argv` in `workdir` with no window, no stdin, its output discarded and its error stream kept in a file there.
|
|
59
|
+
At the deadline the whole process tree is killed; a process that passes `memory_bytes` is killed by the system
|
|
60
|
+
(Windows job memory limit; elsewhere an address-space limit).
|
|
61
|
+
|
|
62
|
+
Parameters: argv, workdir (an existing folder), timeout (seconds), memory_bytes (cap for the tree). Returns:
|
|
63
|
+
`(returncode, stderr_tail)`, the tail being the last 2,000 bytes of the error stream as text. Raises:
|
|
64
|
+
subprocess.TimeoutExpired after killing the tree; OSError when the program cannot start."""
|
|
65
|
+
error_path = os.path.join(workdir, "stderr.txt")
|
|
66
|
+
with open(error_path, "wb") as errors:
|
|
67
|
+
if _IS_WINDOWS:
|
|
68
|
+
returncode = _run_in_job(argv, workdir, timeout, memory_bytes, errors)
|
|
69
|
+
else:
|
|
70
|
+
returncode = _run_in_group(argv, workdir, timeout, memory_bytes, errors)
|
|
71
|
+
return returncode, _stderr_tail(error_path)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _run_in_job(argv, workdir, timeout, memory_bytes, errors):
|
|
75
|
+
import ctypes
|
|
76
|
+
from ctypes import wintypes
|
|
77
|
+
kernel32, job = kill_on_close_job(memory_bytes)
|
|
78
|
+
try:
|
|
79
|
+
proc = subprocess.Popen(argv, cwd=workdir, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=errors,
|
|
80
|
+
creationflags=subprocess.CREATE_NO_WINDOW)
|
|
81
|
+
if not kernel32.AssignProcessToJobObject(wintypes.HANDLE(job), wintypes.HANDLE(proc._handle)):
|
|
82
|
+
proc.kill()
|
|
83
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
84
|
+
try:
|
|
85
|
+
return proc.wait(timeout)
|
|
86
|
+
except subprocess.TimeoutExpired:
|
|
87
|
+
kernel32.TerminateJobObject(wintypes.HANDLE(job), 1)
|
|
88
|
+
proc.wait()
|
|
89
|
+
raise
|
|
90
|
+
finally:
|
|
91
|
+
kernel32.CloseHandle(wintypes.HANDLE(job))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _run_in_group(argv, workdir, timeout, memory_bytes, errors):
|
|
95
|
+
import resource
|
|
96
|
+
import signal
|
|
97
|
+
|
|
98
|
+
def limit_memory():
|
|
99
|
+
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
|
|
100
|
+
|
|
101
|
+
proc = subprocess.Popen(argv, cwd=workdir, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=errors,
|
|
102
|
+
start_new_session=True, preexec_fn=limit_memory)
|
|
103
|
+
try:
|
|
104
|
+
return proc.wait(timeout)
|
|
105
|
+
except subprocess.TimeoutExpired:
|
|
106
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
107
|
+
proc.wait()
|
|
108
|
+
raise
|