@allansantos-dev/smart-tool 0.9.57 → 0.9.65

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,300 @@
1
+ """Documents of the documentation scope that read the project's history and naming: hotspots, hidden coupling,
2
+ dead-code candidates, the onboarding tour, the glossary and the decision candidates found in the git log.
3
+
4
+ Everything is computed from the code graph and `git log`, without a model. The change-coupling and hotspot methods
5
+ follow the published definitions of code-maat (shared revisions over average revisions; revisions times size); no
6
+ code of that GPL tool is used.
7
+ """
8
+ import builtins
9
+ import datetime
10
+ import functools
11
+ import importlib
12
+ import itertools
13
+ import os
14
+ import re
15
+ import sys
16
+ from collections import Counter, defaultdict
17
+
18
+ import affected_tests
19
+ import doc_scope
20
+
21
+ MAX_COMMITS = 3000
22
+ MAX_ROWS = 25
23
+ COUPLING_MAX_CHANGESET = 30
24
+ COUPLING_MIN_REVISIONS = 5
25
+ COUPLING_MIN_SHARED = 5
26
+ COUPLING_MIN_DEGREE = 0.3
27
+ TOUR_STEPS = 8
28
+ TOUR_CALLS = 6
29
+ GLOSSARY_TERMS = 40
30
+ SIMPLE_TERMS = 12
31
+ ADR_CANDIDATES = 25
32
+ ADR_DRAFTS = 5
33
+ _FIELD, _RECORD = "\x1f", "\x1e"
34
+ _MANIFEST = re.compile(r"(^|/)(package\.json|requirements[\w.-]*\.txt|go\.mod|pom\.xml|pyproject\.toml|build\.gradle(\.kts)?)$")
35
+ _DECISION = re.compile(r"\b(decid\w*|replac\w*|switch\w*|migrat\w*|adopt\w*|drop\w*|instead of|introduc\w*|because)\b", re.I)
36
+ _HTTP_VERB = re.compile(r"^do_[A-Z]+$")
37
+ _FRAMEWORK_STYLE = re.compile(r"^(do_[A-Za-z]+|on_\w+|[a-z]+[A-Z]\w*|run|handle|setup|teardown)$")
38
+ _WORDS = re.compile(r"[A-Z]+(?![a-z])|[A-Z]?[a-z]+|\d+")
39
+ _STOP = frozenset("""get set add is has are was the and for not with from into that this per use run main init new old
40
+ make build create update delete remove check handle process helper helpers util utils self cls args kwargs data
41
+ value values item items list dict map key keys name names text str int bool none true false test tests all any""".split())
42
+
43
+
44
+ def history(root):
45
+ """Commits of the project's git history, newest first, with the project-relative files each touched.
46
+
47
+ Returns: (commits, note): commits is a list of {hash, time, subject, body, files}; note is None, or the reason no
48
+ history could be read (not a git repository, git missing), which the page shows in place of the table.
49
+ """
50
+ try:
51
+ raw = affected_tests._git(root, "-c", "core.quotepath=false", "log", "--no-merges", f"-n{MAX_COMMITS}", "--relative",
52
+ "--name-status", "-M", f"--pretty=format:{_RECORD}%h{_FIELD}%ct{_FIELD}%s{_FIELD}%b{_FIELD}", "--", ".")
53
+ except (ValueError, OSError) as error:
54
+ return [], f"No git history could be read: {error}"
55
+ commits, current = [], {}
56
+ for chunk in raw.split(_RECORD)[1:]:
57
+ short, stamp, subject, body, names = chunk.split(_FIELD, 4)
58
+ files = []
59
+ for line in filter(None, names.splitlines()):
60
+ fields = line.split("\t")
61
+ if fields[0][0] == "R" and len(fields) == 3:
62
+ current[fields[1]] = current.get(fields[2], fields[2])
63
+ path = fields[-1]
64
+ path = current.get(path, path)
65
+ if path not in files:
66
+ files.append(path)
67
+ commits.append({"hash": short, "time": int(stamp), "subject": subject, "body": body.strip(), "files": files})
68
+ return commits, (None if commits else "The git history holds no commit that touches this project.")
69
+
70
+
71
+ def _date(stamp):
72
+ return datetime.datetime.fromtimestamp(stamp, datetime.timezone.utc).strftime("%Y-%m-%d")
73
+
74
+
75
+ def _lines(model, path):
76
+ text = model.read(path)
77
+ return text.count("\n") + 1 if text else 0
78
+
79
+
80
+ def hotspots(model, commits):
81
+ """Rows (path, commits, lines, last change) of the code files ranked by commits times lines, highest first."""
82
+ code = {p for p in model.files() if model.kind_of(p) == "code"}
83
+ churn, last = Counter(), {}
84
+ for commit in commits:
85
+ for path in commit["files"]:
86
+ if path in code:
87
+ churn[path] += 1
88
+ last.setdefault(path, commit["time"])
89
+ rows = [(path, count, _lines(model, path), last[path]) for path, count in churn.items()]
90
+ return sorted((r for r in rows if r[2]), key=lambda r: (-r[1] * r[2], r[0]))
91
+
92
+
93
+ def coupling(model, commits):
94
+ """Pairs of files that change in the same commits: (a, b, shared, degree, linked), strongest first.
95
+
96
+ degree is the shared commits over the average commits of the two files; linked tells whether one imports the other.
97
+ Commits that touch more than COUPLING_MAX_CHANGESET files are skipped, as bulk edits couple everything.
98
+ """
99
+ present = set(model.files())
100
+ revisions, shared = Counter(), Counter()
101
+ for commit in commits:
102
+ files = sorted({p for p in commit["files"] if p in present})
103
+ if not files or len(files) > COUPLING_MAX_CHANGESET:
104
+ continue
105
+ revisions.update(files)
106
+ shared.update(itertools.combinations(files, 2))
107
+ imports = {(d["source"], d["target"]) for d in model.graph.get("dependencies") or [] if d.get("target")}
108
+ found = []
109
+ for (a, b), count in shared.items():
110
+ if count < COUPLING_MIN_SHARED or min(revisions[a], revisions[b]) < COUPLING_MIN_REVISIONS:
111
+ continue
112
+ degree = count / ((revisions[a] + revisions[b]) / 2)
113
+ if degree >= COUPLING_MIN_DEGREE:
114
+ found.append((a, b, count, degree, (a, b) in imports or (b, a) in imports))
115
+ return sorted(found, key=lambda r: (-r[3], -r[2], r[0], r[1]))
116
+
117
+
118
+ @functools.lru_cache(maxsize=None)
119
+ def _library_defines(base, method):
120
+ """True when the standard-library class named by the dotted path defines the method."""
121
+ module, _dot, name = base.rpartition(".")
122
+ try:
123
+ owner = getattr(importlib.import_module(module), name) if module else getattr(builtins, name)
124
+ except (ImportError, AttributeError, ValueError):
125
+ return False
126
+ return hasattr(owner, method)
127
+
128
+
129
+ def _framework_method(function, method, project):
130
+ """True when a method may be called by the class it extends, outside the call graph: the standard-library base
131
+ defines it (handle, run) or names an HTTP verb handler (do_GET, which the library looks up by name), or a third-party base is extended with a name in that library's style (Qt's
132
+ eventFilter, an on_ handler). Bases defined in the project are not libraries."""
133
+ for base in function.get("bases", ()):
134
+ root = base.split(".")[0]
135
+ if base.startswith(".") or root in project:
136
+ continue
137
+ if root in sys.stdlib_module_names or base in dir(builtins):
138
+ if _library_defines(base, method) or _HTTP_VERB.match(method):
139
+ return True
140
+ elif _FRAMEWORK_STYLE.match(method):
141
+ return True
142
+ return False
143
+
144
+
145
+ def dead_code(model, entries):
146
+ """Anchors of code functions nothing calls (static calls only) and of those only tests call, in file order. A call
147
+ or a reference (the function handed over as a value) from another function or from module-level code counts as a
148
+ use. A function with a decorator other than staticmethod or classmethod is left out: property, route and
149
+ registration decorators call it from outside the graph, as are the unnamed functions the parser calls `callback` or
150
+ `anonymous`, which run as arguments of the call that holds them."""
151
+ entry = set(entries)
152
+ project = {p.split("/")[0].split(".")[0] for p in model.files()}
153
+ alone, tested = [], []
154
+ for anchor, function in sorted(model.function_of.items(), key=lambda i: (i[1]["path"], i[1]["start"])):
155
+ name = function["name"].split(".")[-1]
156
+ if model.kind[function["path"]] != "code" or anchor in entry or (name.startswith("__") and name.endswith("__")) \
157
+ or name in ("main", "callback", "anonymous") or anchor in model.used_by_module \
158
+ or doc_scope.registered(function) or _framework_method(function, name, project):
159
+ continue
160
+ callers = model.callers[anchor]
161
+ if not callers:
162
+ alone.append(anchor)
163
+ elif all(model.kind[model.function_of[c]["path"]] == "test" for c in callers):
164
+ tested.append(anchor)
165
+ return alone, tested
166
+
167
+
168
+ def terms(model):
169
+ """Words of the code identifiers, as (word, functions using it, example identifiers), most used first."""
170
+ used, examples = defaultdict(set), defaultdict(list)
171
+ for anchor, function in model.function_of.items():
172
+ if model.kind[function["path"]] != "code":
173
+ continue
174
+ name = function["name"].split(".")[-1]
175
+ for word in {w.lower() for w in _WORDS.findall(name)}:
176
+ if len(word) >= 3 and word not in _STOP and not word.isdigit():
177
+ used[word].add(anchor)
178
+ if name not in examples[word] and len(examples[word]) < 3:
179
+ examples[word].append(name)
180
+ ranked = sorted(used, key=lambda w: (-len(used[w]), w))
181
+ return [(w, len(used[w]), examples[w]) for w in ranked if len(used[w]) >= 2]
182
+
183
+
184
+ def adr_candidates(commits):
185
+ """Commits that look like a decision: a subject with decision wording, or a change to a dependency manifest."""
186
+ found = []
187
+ for commit in commits:
188
+ signals = []
189
+ if _DECISION.search(commit["subject"]):
190
+ signals.append("subject")
191
+ if any(_MANIFEST.search(p) for p in commit["files"]):
192
+ signals.append("dependencies")
193
+ if signals:
194
+ found.append({**commit, "signals": signals})
195
+ return found
196
+
197
+
198
+ def _table(lines, head, rows):
199
+ return lines + [f"| {' | '.join(head)} |", f"|{'---|' * len(head)}"] + [f"| {' | '.join(map(str, r))} |" for r in rows]
200
+
201
+
202
+ def hotspots_page(model, head, done, commits, note):
203
+ lines = head(model, "Hotspots")
204
+ if note:
205
+ return done(lines + [note])
206
+ rows = hotspots(model, commits)[:MAX_ROWS]
207
+ lines += [f"Code files ranked by the commits that touched them times their lines (the last {len(commits)} commits): "
208
+ "the files that change most and weigh most are the riskiest to modify.", ""]
209
+ if not rows:
210
+ return done(lines + ["No code file of the index appears in the history."])
211
+ return done(_table(lines, ["File", "Commits", "Lines", "Score", "Last change"],
212
+ [(f"`{p}`", c, n, c * n, _date(t)) for p, c, n, t in rows]))
213
+
214
+
215
+ def coupling_page(model, head, done, commits, note):
216
+ lines = head(model, "Change coupling")
217
+ if note:
218
+ return done(lines + [note])
219
+ pairs = coupling(model, commits)
220
+ hidden = [p for p in pairs if not p[4]]
221
+ lines += ["Files that changed in the same commits without an import between them, which the call graph does not show. "
222
+ f"Degree is the shared commits over the average commits of the two files; shown from {COUPLING_MIN_SHARED} "
223
+ f"shared commits and {int(COUPLING_MIN_DEGREE * 100)}%, in commits of at most {COUPLING_MAX_CHANGESET} files.", ""]
224
+ if hidden:
225
+ lines = _table(lines, ["File", "Changes with", "Shared commits", "Degree"],
226
+ [(f"`{a}`", f"`{b}`", n, f"{int(d * 100)}%") for a, b, n, d, _l in hidden[:MAX_ROWS]]) + [""]
227
+ else:
228
+ lines += ["No pair of files without an import between them changes together often enough.", ""]
229
+ return done(lines + [f"{len(pairs) - len(hidden)} more pair(s) change together but already import each other."])
230
+
231
+
232
+ def dead_code_page(model, head, done, entries):
233
+ lines = head(model, "Dead-code candidates")
234
+ alone, tested = dead_code(model, entries)
235
+ lines += ["Functions no other function or module-level code calls or hands over as a value (Python and Go only; in "
236
+ "other languages a function passed as a value counts as uncalled), from the static call graph only: a call "
237
+ "through a framework, a decorator registration, reflection or a name built at run time is not seen, so "
238
+ "read each one before deleting it.", ""]
239
+ for title, anchors in (("No caller", alone), ("Called only from tests", tested)):
240
+ lines += [f"## {title} ({len(anchors)})", ""]
241
+ if not anchors:
242
+ lines += ["None.", ""]
243
+ continue
244
+ lines += [f"- `{model.function_of[a]['name']}` ({model.function_of[a]['path']}:{model.function_of[a]['start']})"
245
+ + ("" if model.function_of[a]["required"] else " - private") for a in anchors[:100]]
246
+ lines += [f"- ... and {len(anchors) - 100} more" if len(anchors) > 100 else "", ""]
247
+ return done(lines)
248
+
249
+
250
+ def tour_page(model, head, done, entries):
251
+ lines = head(model, "Onboarding tour")
252
+ if not entries:
253
+ return done(lines + ["No entry point found: declare the layers (project_manage action=layers)."])
254
+ lines += ["Read the entry points in this order, highest layer first; each step lists the functions it calls next.", ""]
255
+ for number, anchor in enumerate(entries[:TOUR_STEPS], 1):
256
+ function = model.function_of[anchor]
257
+ calls = [model.function_of[c] for _caller, c in model.flow_edges(anchor)][:TOUR_CALLS]
258
+ title = doc_scope.entry_title(model, anchor, entries)
259
+ lines += [f"{number}. {doc_scope.entry_link(model, anchor, entries, '../functional.md')}"
260
+ + ("" if title.endswith(f"in `{function['path']}`") else f" in `{function['path']}`")
261
+ + (f": {function['doc']}" if function.get("doc") else "")]
262
+ lines += [f" - then `{c['name']}` ({c['path']})" for c in calls]
263
+ if len(entries) > TOUR_STEPS:
264
+ lines += ["", f"{len(entries) - TOUR_STEPS} more entry point(s) in the [functional overview](../functional.md)."]
265
+ return done(lines)
266
+
267
+
268
+ def glossary_page(model, head, done):
269
+ lines = head(model, "Glossary")
270
+ found = terms(model)[:GLOSSARY_TERMS]
271
+ lines += ["Words the code uses in function names, by how many functions use them. Use the same words in the "
272
+ "documentation.", ""]
273
+ if not found:
274
+ return done(lines + ["No word is shared by two functions."])
275
+ return done(_table(lines, ["Word", "Functions", "Examples"], [(f"`{w}`", n, ", ".join(f"`{e}`" for e in ex)) for w, n, ex in found]))
276
+
277
+
278
+ def simple_terms(model):
279
+ """The most used words of the code, for the plain-words summary."""
280
+ return [w for w, _n, _e in terms(model)[:SIMPLE_TERMS]]
281
+
282
+
283
+ def adr_section(commits, note):
284
+ """Lines for the decision-records page: commits that look like decisions with a draft for the latest ones."""
285
+ lines = ["", "## Candidates from the git log", ""]
286
+ if note:
287
+ return lines + [note]
288
+ found = adr_candidates(commits)[:ADR_CANDIDATES]
289
+ if not found:
290
+ return lines + ["No commit with decision wording or a dependency change in the history."]
291
+ lines += ["Commits that look like a decision, by their subject or by a change to a dependency manifest. Each is a "
292
+ "candidate for a record, never a record: only the person who made the decision knows the reasons.", ""]
293
+ lines = _table(lines, ["Date", "Commit", "Subject", "Signal"],
294
+ [(_date(c["time"]), f"`{c['hash']}`", c["subject"].replace("|", "\\|"), ", ".join(c["signals"])) for c in found])
295
+ for commit in found[:ADR_DRAFTS]:
296
+ lines += ["", f"### Draft: {commit['subject']}", "", "```markdown", f"# NNNN. {commit['subject']}", "", "Status: proposed", "",
297
+ "## Context", "", commit["body"] or "To write: why the change was needed.", "", "## Decision", "",
298
+ f"Commit `{commit['hash']}` ({_date(commit['time'])}): {commit['subject']}", "", "## Consequences", "",
299
+ f"To write. The commit touched {len(commit['files'])} file(s).", "```"]
300
+ return lines