@allansantos-dev/smart-tool 0.9.80 → 0.9.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -0
- package/NOTICE +1 -1
- package/README.md +59 -14
- package/agent_redirect.py +32 -0
- package/autostart.py +32 -25
- package/browser_bridge.py +320 -0
- package/browser_control.py +76 -39
- package/browser_service_client.py +1 -7
- package/client_hooks.py +42 -41
- package/code_graph.py +56 -15
- package/code_graph_java.mjs +21 -12
- package/code_health.py +19 -12
- package/code_impact.py +1 -1
- package/codex_config.py +3 -0
- package/config.py +35 -0
- package/daemon_launcher.py +37 -5
- package/doc_candidates.py +63 -0
- package/doc_check.py +9 -3
- package/doc_extras.py +2 -3
- package/doc_insights.py +26 -2
- package/doc_prose.py +59 -11
- package/doc_registry.py +241 -0
- package/doc_scope.py +205 -39
- package/doc_turn.py +205 -0
- package/doc_validate.py +52 -9
- package/document_read.py +4 -2
- package/dossier.py +40 -5
- package/edit_shape.py +119 -0
- package/endpoint_sync.py +3 -21
- package/engine_home.py +95 -0
- package/hook_call.py +132 -0
- package/hook_decision.py +337 -23
- package/hooks/pretool_router.py +36 -29
- package/hooks/turn_end.py +40 -0
- package/index_views.py +5 -0
- package/install.py +90 -36
- package/install_runtime.py +24 -10
- package/lazy_mcp_proxy.py +10 -9
- package/link_check.py +69 -0
- package/local_embedder.py +9 -2
- package/locales/pt-BR.json +72 -4
- package/model_breaker.py +6 -5
- package/model_client.py +1 -1
- package/model_health.py +24 -6
- package/notice_action.py +9 -0
- package/package.json +2 -2
- package/pr_comment.py +94 -0
- package/pr_impact.py +216 -35
- package/process_limits.py +17 -3
- package/project_admission.py +17 -15
- package/project_identity.py +19 -1
- package/project_scan.py +98 -0
- package/removed_callers.py +146 -0
- package/scout_access.py +48 -0
- package/setup_ui.py +33 -4
- package/shell_reads.py +67 -0
- package/smart_tool_daemon.py +149 -39
- package/tray.py +60 -9
- package/update_splash.py +2 -2
- package/venv_launch.py +67 -0
- package/version.py +1 -1
- package/web_adapters/node/engine-home.mjs +42 -0
- package/web_adapters/node/package-lock.json +13 -0
- package/web_adapters/node/package.json +1 -0
- package/web_document_graph.py +12 -1
package/pr_impact.py
CHANGED
|
@@ -2,27 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
It joins what Smart Tool already computes: the functions the diff touches and the tests to run (affected_tests), the
|
|
4
4
|
callers of each touched function and the tests that reach it (code_impact), the documentation anchors the change leaves
|
|
5
|
-
out of date (doc_scope), the files that usually change with the diff's files and are missing from it
|
|
6
|
-
|
|
5
|
+
out of date (doc_scope), the files that usually change with the diff's files and are missing from it, the import cycles
|
|
6
|
+
through the changed files and the changed files that are hotspots (code_health), the new functions no caller or test reaches (code_health), the callers that still name a function
|
|
7
|
+
the diff removed or renamed and the callers of those whose signature changed (removed_callers), the decision notes that name a touched function (dossier) and the notes the
|
|
8
|
+
edit hook would have added to the whole diff (hook_decision).
|
|
7
9
|
Nothing is published: the agent pastes the report where it belongs.
|
|
8
10
|
"""
|
|
9
11
|
import os
|
|
10
|
-
import re
|
|
11
12
|
|
|
12
13
|
import affected_tests
|
|
13
14
|
import code_health
|
|
14
15
|
import code_impact
|
|
15
16
|
import config
|
|
17
|
+
import doc_check
|
|
16
18
|
import doc_scope
|
|
17
19
|
import dossier
|
|
18
20
|
import hook_decision
|
|
21
|
+
import removed_callers
|
|
19
22
|
|
|
20
23
|
MAX_FUNCTIONS = 15
|
|
21
24
|
MAX_CALLERS = 8
|
|
22
25
|
TEST_DEPTH = 3
|
|
23
26
|
MAX_CHECKED_FILES = 50
|
|
24
27
|
MAX_COUPLED = 10
|
|
25
|
-
|
|
28
|
+
MAX_HOTSPOTS = 10
|
|
29
|
+
ALL_ROWS = 100000
|
|
26
30
|
|
|
27
31
|
|
|
28
32
|
def _callers(root, label, view_id):
|
|
@@ -34,34 +38,119 @@ def _callers(root, label, view_id):
|
|
|
34
38
|
return found["callers"], found["counts"]["callers"], found["counts"]["tests"]
|
|
35
39
|
|
|
36
40
|
|
|
37
|
-
def
|
|
38
|
-
"""The
|
|
41
|
+
def _versions(root, base, changes):
|
|
42
|
+
"""The text of each changed file at the base and now.
|
|
39
43
|
|
|
40
44
|
Parameters: root (str) project folder; base (str) git revision; changes (dict) the `affected_tests.affected` answer.
|
|
41
|
-
Returns:
|
|
45
|
+
Returns: list of (project-relative path, absolute path, text at the base or None when the file is new, text now or None
|
|
46
|
+
when the file is gone); files that cannot be read as text are left out.
|
|
42
47
|
"""
|
|
43
48
|
paths = sorted({*changes["changed"]["code"], *changes["changed"]["tests"], *changes["changed"]["other"]})
|
|
44
|
-
|
|
49
|
+
found = []
|
|
45
50
|
for rel in paths[:MAX_CHECKED_FILES]:
|
|
46
51
|
full = os.path.join(root, *rel.split("/"))
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
after = None
|
|
53
|
+
if os.path.isfile(full):
|
|
54
|
+
try:
|
|
55
|
+
with open(full, encoding="utf-8") as stream:
|
|
56
|
+
after = stream.read()
|
|
57
|
+
except (OSError, UnicodeDecodeError):
|
|
58
|
+
continue
|
|
54
59
|
try:
|
|
55
60
|
before = affected_tests._git(root, "show", f"{base}:./{rel}")
|
|
56
61
|
except ValueError:
|
|
57
62
|
before = None
|
|
58
|
-
|
|
63
|
+
found.append((rel, full, before, after))
|
|
64
|
+
return found
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _hook_notes(versions):
|
|
68
|
+
"""The edit hook's notes over the changed files of the diff, which commits made without the hook never got.
|
|
69
|
+
|
|
70
|
+
Parameters: versions (list) `_versions` of the diff.
|
|
71
|
+
Returns: tuple (str notes, "" when nothing is worth saying; str problem, "" when the checks ran).
|
|
72
|
+
"""
|
|
73
|
+
triples = [(full, before, after) for _rel, full, before, after in versions if after is not None]
|
|
59
74
|
try:
|
|
60
75
|
return hook_decision.review_diff(triples, config.load_config()), ""
|
|
61
76
|
except (OSError, ValueError, RuntimeError) as exc:
|
|
62
77
|
return "", f"the edit checks could not run: {exc}"
|
|
63
78
|
|
|
64
79
|
|
|
80
|
+
def _reviewed(root, versions, changed_code, review, key, what):
|
|
81
|
+
"""The answers of `review` over the changed code files that have something under `key`, or the reason it could not run.
|
|
82
|
+
|
|
83
|
+
Returns: tuple (list of answers; str problem, "" when the check ran).
|
|
84
|
+
"""
|
|
85
|
+
answers = []
|
|
86
|
+
try:
|
|
87
|
+
for rel, _full, before, after in versions:
|
|
88
|
+
if rel in changed_code:
|
|
89
|
+
answer = review(root, rel, before, after or "")
|
|
90
|
+
if answer[key]:
|
|
91
|
+
answers.append(answer)
|
|
92
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
93
|
+
return answers, f"the callers of {what} could not be read: {exc}"
|
|
94
|
+
return answers, ""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _removed_callers(root, versions, changed_code):
|
|
98
|
+
"""The callers that still name a function the diff removed or renamed, in the changed code files.
|
|
99
|
+
|
|
100
|
+
Returns: tuple (list of answers of `removed_callers.review`, one per file that lost a function; str problem, "" when the
|
|
101
|
+
check ran).
|
|
102
|
+
"""
|
|
103
|
+
return _reviewed(root, versions, changed_code, removed_callers.review, "removed", "removed functions")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _signature_callers(root, versions, changed_code):
|
|
107
|
+
"""The callers outside the file of the functions whose signature the diff changed, in the changed code files.
|
|
108
|
+
|
|
109
|
+
Returns: tuple (list of answers of `removed_callers.signature_review`, one per file with a changed signature; str problem,
|
|
110
|
+
"" when the check ran).
|
|
111
|
+
"""
|
|
112
|
+
return _reviewed(root, versions, changed_code, removed_callers.signature_review, "changed", "functions with a changed signature")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _also_defined(item):
|
|
116
|
+
return f" A function named {item['name'].rpartition('.')[2]} also exists in {', '.join(item['also_defined_in'])}." \
|
|
117
|
+
if item["also_defined_in"] else ""
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _callers_section(answers, problem, title, key, finding, quiet_text):
|
|
121
|
+
"""The Markdown lines of one caller section, none when there is nothing to say.
|
|
122
|
+
|
|
123
|
+
Parameters: answers (list) of `removed_callers` answers; problem (str) why the check could not run, "" when it ran;
|
|
124
|
+
title (str) the `###` heading; key (str) the answer key holding the names of the functions concerned; finding (str)
|
|
125
|
+
format of a caller line over `item` and `sites`; quiet_text (str) format of the line for names with no caller.
|
|
126
|
+
"""
|
|
127
|
+
if not answers and not problem:
|
|
128
|
+
return []
|
|
129
|
+
lines = [f"### {title}", ""]
|
|
130
|
+
for answer in answers:
|
|
131
|
+
for item in answer["findings"]:
|
|
132
|
+
lines.append(finding.format(item=item, sites=removed_callers.sites_text(item), also=_also_defined(item)))
|
|
133
|
+
quiet = [name for name in answer[key] if name not in {item["name"] for item in answer["findings"]}]
|
|
134
|
+
if quiet:
|
|
135
|
+
lines.append(quiet_text.format(names=", ".join(f"`{name}`" for name in quiet)))
|
|
136
|
+
lines += list(dict.fromkeys(answer["note"] for answer in answers if answer["note"]))
|
|
137
|
+
return lines + ([problem] if problem else []) + [""]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _removed_lines(answers, problem):
|
|
141
|
+
"""The Markdown lines of the section on removed or renamed functions, none when the diff removes no function."""
|
|
142
|
+
return _callers_section(answers, problem, "Callers of removed or renamed functions", "removed",
|
|
143
|
+
"- `{item[name]}` ({item[path]}) is gone and is still called from {sites}.{also}",
|
|
144
|
+
"No caller was found for the removed {names}.")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _signature_lines(answers, problem):
|
|
148
|
+
"""The Markdown lines of the section on functions whose signature changed, none when the diff changes no signature."""
|
|
149
|
+
return _callers_section(answers, problem, "Callers of functions whose signature changed", "changed",
|
|
150
|
+
"- `{item[name]}` ({item[path]}) has a new signature and is called from {sites} outside the file.{also}",
|
|
151
|
+
"No caller outside the file was found for {names}, whose signature changed.")
|
|
152
|
+
|
|
153
|
+
|
|
65
154
|
def _coupled(root, view_id, changes):
|
|
66
155
|
"""Files that usually change with the diff's files but are not in it, or the reason none could be computed."""
|
|
67
156
|
paths = sorted({*changes["changed"]["code"], *changes["changed"]["tests"], *changes["changed"]["other"]})
|
|
@@ -72,33 +161,113 @@ def _coupled(root, view_id, changes):
|
|
|
72
161
|
return found["files"], f"The change coupling was not read: {found['note']}" if found["note"] else None
|
|
73
162
|
|
|
74
163
|
|
|
75
|
-
def
|
|
76
|
-
"""
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
164
|
+
def _cycles(root, view_id, changed_code):
|
|
165
|
+
"""Import cycles that contain a changed code file, or the reason none could be computed.
|
|
166
|
+
|
|
167
|
+
Returns: tuple (list of `code_health.import_cycles` entries; str note, what the graph cut; str problem, "" when the
|
|
168
|
+
cycles were read).
|
|
169
|
+
"""
|
|
170
|
+
try:
|
|
171
|
+
found = code_health.import_cycles(root, view_id, limit=ALL_ROWS)
|
|
172
|
+
except (ValueError, RuntimeError) as exc:
|
|
173
|
+
return [], "", f"The import cycles were not read: {exc}"
|
|
174
|
+
return [c for c in found["cycles"] if changed_code.intersection(c["files"])], found["note"], ""
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _hotspots(root, view_id, changed_code):
|
|
178
|
+
"""Changed code files among the hotspots of the project, or the reason none could be computed.
|
|
179
|
+
|
|
180
|
+
Returns: tuple (list of `code_health.hotspots` rows, riskiest first; str problem, "" when the history was read).
|
|
181
|
+
"""
|
|
182
|
+
try:
|
|
183
|
+
found = code_health.hotspots(root, view_id, limit=ALL_ROWS)
|
|
184
|
+
except (ValueError, RuntimeError) as exc:
|
|
185
|
+
return [], f"The hotspots were not read: {exc}"
|
|
186
|
+
if found["note"]:
|
|
187
|
+
return [], f"The hotspots were not read: {found['note']}"
|
|
188
|
+
return [f for f in found["files"] if f["path"] in changed_code][:MAX_HOTSPOTS], ""
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _added_functions(versions, changed_code):
|
|
192
|
+
"""The (path, qualified name) pairs of the functions the diff adds: those a changed code file defines now and did not at
|
|
193
|
+
the base, every function of a new file included."""
|
|
194
|
+
added = set()
|
|
195
|
+
for rel, _full, before, after in versions:
|
|
196
|
+
if rel in changed_code and after is not None and doc_check.supported(rel):
|
|
197
|
+
known = {f["name"] for f in doc_check.functions(rel, before)} if before is not None else set()
|
|
198
|
+
added.update((rel, f["name"]) for f in doc_check.functions(rel, after) if f["name"] not in known)
|
|
199
|
+
return added
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _new_dead(root, view_id, versions, changed_code):
|
|
203
|
+
"""The functions the diff adds that no function and no test calls, from `code_health.dead_code`.
|
|
204
|
+
|
|
205
|
+
Returns: tuple (list of `dead_code` entries; str note, what the graph cut; str problem, "" when the check ran). Nothing is
|
|
206
|
+
asked of the graph when the diff adds no function.
|
|
207
|
+
"""
|
|
208
|
+
added = _added_functions(versions, changed_code)
|
|
209
|
+
if not added:
|
|
210
|
+
return [], "", ""
|
|
211
|
+
try:
|
|
212
|
+
found = code_health.dead_code(root, view_id, limit=ALL_ROWS)
|
|
213
|
+
except (ValueError, RuntimeError) as exc:
|
|
214
|
+
return [], "", f"The new functions without a caller were not read: {exc}"
|
|
215
|
+
return [f for f in found["no_caller"] if (f["path"], f["name"]) in added], found["graph_note"], ""
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _new_dead_lines(dead, note, problem):
|
|
219
|
+
"""The Markdown section on new functions nothing calls; left out when the diff adds none or all have a caller, unless
|
|
220
|
+
the graph was cut or the check could not run, which are said."""
|
|
221
|
+
if not (dead or note or problem):
|
|
222
|
+
return []
|
|
223
|
+
lines = ["### New functions nothing calls", ""]
|
|
224
|
+
lines += [f"- `{f['name']}` ({f['path']}:{f['line']})" + ("" if f["public"] else " - private") for f in dead]
|
|
225
|
+
if dead:
|
|
226
|
+
lines.append("Candidates from the static call graph (Python and Go only): a call through a framework, a decorator or a "
|
|
227
|
+
"name built at run time is not seen.")
|
|
228
|
+
elif not problem:
|
|
229
|
+
lines.append(f"No new function without a caller was found. {note}".rstrip())
|
|
230
|
+
note = ""
|
|
231
|
+
lines += [note] if note else []
|
|
232
|
+
return lines + ([problem] if problem else []) + [""]
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _shape_lines(cycles, cycles_note, cycles_problem, hotspots, hotspots_problem, changed_code):
|
|
236
|
+
"""The Markdown sections on import cycles and hotspots of the changed files; a section with nothing to say is left out
|
|
237
|
+
unless the graph was cut or the check could not run, which are said."""
|
|
238
|
+
lines = []
|
|
239
|
+
if cycles or cycles_note or cycles_problem:
|
|
240
|
+
lines += ["### Import cycles through the changed files", ""]
|
|
241
|
+
lines += [f"- {c['size']} files import each other, directly or through a chain: "
|
|
242
|
+
+ ", ".join(f"`{f}`" for f in c["files"]) + "; changed: "
|
|
243
|
+
+ ", ".join(f"`{f}`" for f in c["files"] if f in changed_code) for c in cycles]
|
|
244
|
+
if not cycles and not cycles_problem:
|
|
245
|
+
lines.append(f"No cycle through the changed files was found. {cycles_note}")
|
|
246
|
+
lines += [cycles_problem] if cycles_problem else []
|
|
247
|
+
lines.append("")
|
|
248
|
+
if hotspots or hotspots_problem:
|
|
249
|
+
lines += ["### Changed files that are hotspots", ""]
|
|
250
|
+
lines += [f"- `{f['path']}`: {f['commits']} commits, {f['lines']} lines, last changed {f['last_change']}" for f in hotspots]
|
|
251
|
+
lines += [hotspots_problem] if hotspots_problem else []
|
|
252
|
+
lines.append("")
|
|
253
|
+
return lines
|
|
83
254
|
|
|
84
255
|
|
|
85
256
|
def _decisions(root, labels):
|
|
86
|
-
"""Decision notes and commit messages that name each touched function
|
|
87
|
-
|
|
257
|
+
"""Decision notes and commit messages that name each touched function (`dossier.notes_about`, whose rule keeps a
|
|
258
|
+
single word out of ordinary prose).
|
|
88
259
|
|
|
89
260
|
Returns: tuple (dict label -> list of hits, str problem, "" when the search ran whole).
|
|
90
261
|
"""
|
|
91
262
|
found, reasons = {}, []
|
|
92
263
|
for label in labels:
|
|
93
|
-
|
|
94
|
-
if not
|
|
264
|
+
rel, _, qualified = label.partition("::")
|
|
265
|
+
if not qualified:
|
|
95
266
|
continue
|
|
96
|
-
answer = dossier.
|
|
267
|
+
answer = dossier.notes_about(root, rel, qualified)
|
|
97
268
|
reasons += [reason for reason in answer["reasons"] if reason not in reasons]
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
if hits:
|
|
101
|
-
found[label] = hits[:MAX_NOTES_PER_FUNCTION]
|
|
269
|
+
if answer["hits"]:
|
|
270
|
+
found[label] = answer["hits"]
|
|
102
271
|
return found, f"The notes search was cut: {'; '.join(reasons)}" if reasons else ""
|
|
103
272
|
|
|
104
273
|
|
|
@@ -122,9 +291,15 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
122
291
|
callers = {label: _callers(root, label, view_id) for label in touched}
|
|
123
292
|
stale, stale_note = _stale(root, view_id, changed_code)
|
|
124
293
|
untested = [label for label in touched if callers[label] is None or callers[label][2] == 0]
|
|
125
|
-
|
|
294
|
+
versions = _versions(root, base, changes)
|
|
295
|
+
notes, notes_problem = _hook_notes(versions)
|
|
296
|
+
removed, removed_problem = _removed_callers(root, versions, changed_code)
|
|
297
|
+
resigned, resigned_problem = _signature_callers(root, versions, changed_code)
|
|
126
298
|
coupled, coupled_note = _coupled(root, view_id, changes)
|
|
127
299
|
decisions, decisions_note = _decisions(root, touched)
|
|
300
|
+
cycles, cycles_note, cycles_problem = _cycles(root, view_id, changed_code) if changed_code else ([], "", "")
|
|
301
|
+
hotspots, hotspots_problem = _hotspots(root, view_id, changed_code) if changed_code else ([], "")
|
|
302
|
+
new_dead, new_dead_note, new_dead_problem = _new_dead(root, view_id, versions, changed_code)
|
|
128
303
|
lines = [f"## Impact of the change against `{base}`", "",
|
|
129
304
|
f"{changes['counts']['changed_files']} changed file(s), {len(changes['touched_functions'])} touched "
|
|
130
305
|
f"function(s), {changes['counts']['tests']} test file(s) to run.", ""]
|
|
@@ -146,6 +321,8 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
146
321
|
lines += ["### Touched functions no test reaches", ""]
|
|
147
322
|
lines += [f"- `{label}`" for label in untested] if untested else ["Every touched function is reached by a test."]
|
|
148
323
|
lines.append("")
|
|
324
|
+
lines += _removed_lines(removed, removed_problem)
|
|
325
|
+
lines += _signature_lines(resigned, resigned_problem)
|
|
149
326
|
lines += ["### Documentation left out of date", ""]
|
|
150
327
|
if stale_note:
|
|
151
328
|
lines += [stale_note, ""]
|
|
@@ -169,6 +346,8 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
169
346
|
for f in coupled] + [""]
|
|
170
347
|
else:
|
|
171
348
|
lines += ["No file that usually changes with these is missing from the diff.", ""]
|
|
349
|
+
lines += _shape_lines(cycles, cycles_note, cycles_problem, hotspots, hotspots_problem, changed_code)
|
|
350
|
+
lines += _new_dead_lines(new_dead, new_dead_note, new_dead_problem)
|
|
172
351
|
lines += ["### Tests to run", ""]
|
|
173
352
|
if changes["tests"]:
|
|
174
353
|
lines += [f"- `{t['path']}`: {t['why']}" for t in changes["tests"]]
|
|
@@ -184,6 +363,8 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
184
363
|
lines += ["", changes["note"]]
|
|
185
364
|
return {"base": base, "markdown": "\n".join(lines).rstrip("\n") + "\n", "touched": touched,
|
|
186
365
|
"callers": {label: (reached[1] if reached else 0) for label, reached in callers.items()},
|
|
187
|
-
"untested": untested, "coupled": coupled, "decisions": decisions, "edit_checks": notes or notes_problem,
|
|
366
|
+
"untested": untested, "coupled": coupled, "cycles": cycles, "hotspots": hotspots, "new_without_caller": new_dead, "decisions": decisions, "edit_checks": notes or notes_problem,
|
|
367
|
+
"removed_callers": [item for answer in removed for item in answer["findings"]],
|
|
368
|
+
"signature_callers": [item for answer in resigned for item in answer["findings"]],
|
|
188
369
|
"stale_anchors": stale, "stale_note": stale_note, "tests": changes["tests"],
|
|
189
370
|
"commands": changes["commands"], "run_all": changes["run_all"], "counts": changes["counts"]}
|
package/process_limits.py
CHANGED
|
@@ -47,13 +47,27 @@ def kill_on_close_job(job_memory_bytes=None):
|
|
|
47
47
|
return kernel32, job
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def
|
|
50
|
+
def file_tail(path, limit=STDERR_TAIL_BYTES):
|
|
51
|
+
"""The last `limit` bytes of the file at `path`, decoded and stripped. Raises OSError when it cannot be read."""
|
|
51
52
|
with open(path, "rb") as stream:
|
|
52
53
|
stream.seek(0, os.SEEK_END)
|
|
53
|
-
stream.seek(max(0, stream.tell() -
|
|
54
|
+
stream.seek(max(0, stream.tell() - limit))
|
|
54
55
|
return stream.read().decode("utf-8", "replace").strip()
|
|
55
56
|
|
|
56
57
|
|
|
58
|
+
def bind_to_kill_on_close_job(job, proc):
|
|
59
|
+
"""Puts `proc` in the kill-on-close job `job` ((kernel32, handle) from `kill_on_close_job`; created when None) and
|
|
60
|
+
returns the job. Raises OSError when the system refuses: an unbound child would outlive its owner."""
|
|
61
|
+
import ctypes
|
|
62
|
+
from ctypes import wintypes
|
|
63
|
+
if job is None:
|
|
64
|
+
job = kill_on_close_job()
|
|
65
|
+
kernel32, handle = job
|
|
66
|
+
if not kernel32.AssignProcessToJobObject(wintypes.HANDLE(handle), wintypes.HANDLE(proc._handle)):
|
|
67
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
68
|
+
return job
|
|
69
|
+
|
|
70
|
+
|
|
57
71
|
def run_capped(argv, workdir, timeout, memory_bytes):
|
|
58
72
|
"""Runs `argv` in `workdir` with no window, no stdin, its output discarded and its error stream kept in a file there.
|
|
59
73
|
At the deadline the whole process tree is killed; a process that passes `memory_bytes` is killed by the system
|
|
@@ -68,7 +82,7 @@ def run_capped(argv, workdir, timeout, memory_bytes):
|
|
|
68
82
|
returncode = _run_in_job(argv, workdir, timeout, memory_bytes, errors)
|
|
69
83
|
else:
|
|
70
84
|
returncode = _run_in_group(argv, workdir, timeout, memory_bytes, errors)
|
|
71
|
-
return returncode,
|
|
85
|
+
return returncode, file_tail(error_path)
|
|
72
86
|
|
|
73
87
|
|
|
74
88
|
def _run_in_job(argv, workdir, timeout, memory_bytes, errors):
|
package/project_admission.py
CHANGED
|
@@ -8,6 +8,7 @@ longer exist, are junk (`junk_reason`)."""
|
|
|
8
8
|
import os
|
|
9
9
|
import tempfile
|
|
10
10
|
|
|
11
|
+
import i18n
|
|
11
12
|
import project_identity
|
|
12
13
|
import project_store
|
|
13
14
|
import redirect_rule
|
|
@@ -81,25 +82,25 @@ def refusal(root):
|
|
|
81
82
|
norm = _norm(real)
|
|
82
83
|
scratch = _scratch_roots()
|
|
83
84
|
if os.path.dirname(real) == real:
|
|
84
|
-
return "a drive or filesystem root"
|
|
85
|
+
return i18n.N_("a drive or filesystem root")
|
|
85
86
|
if norm in _personal_folders():
|
|
86
|
-
return "a personal folder, not a project"
|
|
87
|
+
return i18n.N_("a personal folder, not a project")
|
|
87
88
|
if norm == scratch[0]:
|
|
88
|
-
return "the temporary folder"
|
|
89
|
+
return i18n.N_("the temporary folder")
|
|
89
90
|
home = _norm(os.path.expanduser("~"))
|
|
90
91
|
if _inside(norm, home) and norm != home and os.path.relpath(norm, home).split(os.sep)[0].startswith("."):
|
|
91
|
-
return "inside a tool's hidden folder in the home folder"
|
|
92
|
+
return i18n.N_("inside a tool's hidden folder in the home folder")
|
|
92
93
|
if any(_inside(norm, base) for base in _system_roots()):
|
|
93
|
-
return "a system or application folder"
|
|
94
|
+
return i18n.N_("a system or application folder")
|
|
94
95
|
if (any(_inside(norm, base) for base in _env_folders("APPDATA", "LOCALAPPDATA", "LOCALAPPDATA_LOW"))
|
|
95
96
|
and not any(_inside(norm, base) for base in scratch)):
|
|
96
|
-
return "application data"
|
|
97
|
+
return i18n.N_("application data")
|
|
97
98
|
folder = next((part for part in real.replace("\\", "/").split("/") if part.lower() in TOOL_FOLDERS), None)
|
|
98
99
|
if folder:
|
|
99
|
-
return
|
|
100
|
+
return i18n.N_("a build or dependency folder ({folder})").format(folder=folder)
|
|
100
101
|
java = _env_folders("JAVA_HOME")
|
|
101
102
|
if any(_inside(norm, base) for base in java) or any(_is_java_home(path) for path in _ancestors(real)):
|
|
102
|
-
return "a Java home"
|
|
103
|
+
return i18n.N_("a Java home")
|
|
103
104
|
return None
|
|
104
105
|
|
|
105
106
|
|
|
@@ -110,7 +111,7 @@ def temporary_refusal(root):
|
|
|
110
111
|
norm = _norm(project_identity.display_root(root))
|
|
111
112
|
if _inside(norm, _norm(project_identity.display_root(redirect_rule.system_temp()))) and not any(
|
|
112
113
|
_inside(norm, base) for base in _env_folders("SMART_TOOL_SCRATCH")):
|
|
113
|
-
return "inside the system temporary folder"
|
|
114
|
+
return i18n.N_("inside the system temporary folder")
|
|
114
115
|
return None
|
|
115
116
|
|
|
116
117
|
|
|
@@ -150,18 +151,19 @@ def admit(path, origin=None):
|
|
|
150
151
|
return root
|
|
151
152
|
reason = refusal(root) or temporary_refusal(root)
|
|
152
153
|
if reason:
|
|
153
|
-
raise ValueError(
|
|
154
|
-
|
|
154
|
+
raise ValueError(i18n.t("Not indexing {root}: it is {reason}. A project is the folder where the session started; "
|
|
155
|
+
"pass that folder as project_root.", root=root, reason=i18n.t(reason)))
|
|
155
156
|
if origin:
|
|
156
157
|
home, _held = resolve(origin)
|
|
157
158
|
home_reason = refusal(home)
|
|
158
159
|
if home_reason:
|
|
159
|
-
raise ValueError(
|
|
160
|
-
|
|
160
|
+
raise ValueError(i18n.t("Not indexing {root}: the session started in {home}, which is {home_reason}, not a "
|
|
161
|
+
"project. Start the session inside the project folder.",
|
|
162
|
+
root=root, home=home, home_reason=i18n.t(home_reason)))
|
|
161
163
|
if _inside(_norm(root), _norm(home)):
|
|
162
164
|
return home
|
|
163
|
-
raise ValueError(
|
|
164
|
-
|
|
165
|
+
raise ValueError(i18n.t("Not indexing {root}: the session started in the project {home}, and a new project is "
|
|
166
|
+
"indexed only from the session that started in it.", root=root, home=home))
|
|
165
167
|
return root
|
|
166
168
|
|
|
167
169
|
|
package/project_identity.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
"""Identidade única de raízes e contenção de caminhos, inclusive junctions Windows."""
|
|
2
2
|
import hashlib
|
|
3
3
|
import os
|
|
4
|
+
import stat
|
|
4
5
|
import time
|
|
5
6
|
|
|
7
|
+
_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
|
6
8
|
RESOLVED_TTL_S = 3
|
|
7
9
|
RESOLVED_MAX = 4096
|
|
8
10
|
_resolved = {}
|
|
@@ -43,10 +45,26 @@ def legacy_ids(root):
|
|
|
43
45
|
for path in variants))
|
|
44
46
|
|
|
45
47
|
|
|
48
|
+
def _canonical_file(candidate):
|
|
49
|
+
"""canonical_root of a file inside a folder: the folder's resolution (cached like a root) plus the file name, when
|
|
50
|
+
one lstat shows the file itself is no symlink or junction. Every file of an index pass was resolved on its own,
|
|
51
|
+
the costliest call of a pass (one handle per file); this keeps one resolution per folder. A path that cannot be
|
|
52
|
+
read is resolved as before."""
|
|
53
|
+
absolute = os.path.abspath(os.path.expanduser(os.fspath(candidate)))
|
|
54
|
+
folder, name = os.path.split(absolute)
|
|
55
|
+
try:
|
|
56
|
+
leaf = os.lstat(absolute)
|
|
57
|
+
except OSError:
|
|
58
|
+
return canonical_root(absolute)
|
|
59
|
+
if not name or stat.S_ISLNK(leaf.st_mode) or getattr(leaf, "st_file_attributes", 0) & _REPARSE_POINT:
|
|
60
|
+
return canonical_root(absolute)
|
|
61
|
+
return os.path.normcase(os.path.join(display_root(folder), name))
|
|
62
|
+
|
|
63
|
+
|
|
46
64
|
def within_root(root, candidate):
|
|
47
65
|
try:
|
|
48
66
|
base = canonical_root(root)
|
|
49
|
-
resolved =
|
|
67
|
+
resolved = _canonical_file(candidate)
|
|
50
68
|
return os.path.commonpath([base, resolved]) == base
|
|
51
69
|
except (OSError, ValueError, TypeError):
|
|
52
70
|
return False
|
package/project_scan.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Whole-project scans of the files git tracks, read from disk: credentials committed earlier or sitting in a `.env`
|
|
2
|
+
file (the edit hook only sees the lines an edit writes, and the indexer skips secret files) and relative links of HTML
|
|
3
|
+
and Markdown files that point to nothing. Nothing is written and no value of a credential is returned.
|
|
4
|
+
"""
|
|
5
|
+
import os
|
|
6
|
+
import subprocess
|
|
7
|
+
|
|
8
|
+
import credential_check
|
|
9
|
+
import link_check
|
|
10
|
+
|
|
11
|
+
DEFAULT_LIMIT = 25
|
|
12
|
+
MAX_BYTES = 2_000_000
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def tracked_files(root):
|
|
16
|
+
"""Project-relative paths of the files git tracks under `root` that exist on disk.
|
|
17
|
+
|
|
18
|
+
Parameters: root (str) project folder.
|
|
19
|
+
Returns: list of str with forward slashes, in git's order.
|
|
20
|
+
Raises: ValueError when git cannot list the folder (not a repository, git missing, timeout).
|
|
21
|
+
"""
|
|
22
|
+
try:
|
|
23
|
+
listed = subprocess.run(["git", "ls-files", "-z"], cwd=root, capture_output=True, timeout=30,
|
|
24
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
25
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
26
|
+
raise ValueError(f"git ls-files could not run in {root}: {exc}") from exc
|
|
27
|
+
if listed.returncode:
|
|
28
|
+
raise ValueError(f"git ls-files failed in {root}: {listed.stderr.decode('utf-8', 'replace').strip()[:300]}")
|
|
29
|
+
return [rel for rel in listed.stdout.decode("utf-8", "replace").split("\0")
|
|
30
|
+
if rel and os.path.isfile(os.path.join(root, rel))]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _text(path):
|
|
34
|
+
"""Contents of a file as UTF-8 text; None for a binary, oversized, non-UTF-8 or unreadable file."""
|
|
35
|
+
try:
|
|
36
|
+
if os.path.getsize(path) > MAX_BYTES:
|
|
37
|
+
return None
|
|
38
|
+
with open(path, "rb") as stream:
|
|
39
|
+
data = stream.read()
|
|
40
|
+
return None if b"\0" in data else data.decode("utf-8")
|
|
41
|
+
except (OSError, UnicodeDecodeError):
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def tracked_texts(root, extensions=None):
|
|
46
|
+
"""The tracked files of the project with their text, one at a time.
|
|
47
|
+
|
|
48
|
+
Parameters: root (str) project folder; extensions (tuple of str or None) lowercase suffixes to keep, all files when None.
|
|
49
|
+
Returns: iterator of (path, text); text is None for a file that is not readable text.
|
|
50
|
+
Raises: ValueError when git cannot list the folder.
|
|
51
|
+
"""
|
|
52
|
+
for rel in tracked_files(root):
|
|
53
|
+
if extensions is None or rel.lower().endswith(extensions):
|
|
54
|
+
yield rel, _text(os.path.join(root, rel))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def credentials(root, limit=DEFAULT_LIMIT):
|
|
58
|
+
"""Credential shapes in every tracked text file, with the file and line and never the value.
|
|
59
|
+
|
|
60
|
+
Parameters: root (str) project folder; limit (int) findings returned.
|
|
61
|
+
Returns: dict {"findings": [{"path", "line", "kind"}], "total": int, "by_kind": {kind: count}, "files_checked": int,
|
|
62
|
+
"files_skipped": int}; skipped files are binary, over `MAX_BYTES` or not UTF-8.
|
|
63
|
+
Raises: ValueError when git cannot list the folder.
|
|
64
|
+
"""
|
|
65
|
+
findings, checked, skipped = [], 0, 0
|
|
66
|
+
for rel, text in tracked_texts(root):
|
|
67
|
+
if text is None:
|
|
68
|
+
skipped += 1
|
|
69
|
+
continue
|
|
70
|
+
checked += 1
|
|
71
|
+
findings += [{"path": rel, **item} for item in credential_check.review(None, text)]
|
|
72
|
+
by_kind = {}
|
|
73
|
+
for item in findings:
|
|
74
|
+
by_kind[item["kind"]] = by_kind.get(item["kind"], 0) + 1
|
|
75
|
+
return {"findings": findings[:limit], "total": len(findings), "by_kind": by_kind, "files_checked": checked,
|
|
76
|
+
"files_skipped": skipped}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def links(root, limit=DEFAULT_LIMIT):
|
|
80
|
+
"""Relative links of the tracked HTML and Markdown files that point to a missing file, and the links that could not
|
|
81
|
+
be decided (root-relative, template expressions, pages with a base element, unparsable files).
|
|
82
|
+
|
|
83
|
+
Parameters: root (str) project folder; limit (int) rows returned in each list.
|
|
84
|
+
Returns: dict {"broken": [{"path", "line", "link"}], "unchecked": [{"path", "line", "link", "why"}],
|
|
85
|
+
"total_broken": int, "total_unchecked": int, "files_checked": int, "files_skipped": int}.
|
|
86
|
+
Raises: ValueError when git cannot list the folder.
|
|
87
|
+
"""
|
|
88
|
+
broken, unchecked, checked, skipped = [], [], 0, 0
|
|
89
|
+
for rel, text in tracked_texts(root, link_check.EXTENSIONS):
|
|
90
|
+
if text is None:
|
|
91
|
+
skipped += 1
|
|
92
|
+
continue
|
|
93
|
+
checked += 1
|
|
94
|
+
found = link_check.review(os.path.join(root, rel), None, text)
|
|
95
|
+
broken += [{"path": rel, **item} for item in found["broken"]]
|
|
96
|
+
unchecked += [{"path": rel, **item} for item in found["unchecked"]]
|
|
97
|
+
return {"broken": broken[:limit], "unchecked": unchecked[:limit], "total_broken": len(broken),
|
|
98
|
+
"total_unchecked": len(unchecked), "files_checked": checked, "files_skipped": skipped}
|