@allansantos-dev/smart-tool 0.9.57 → 0.9.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +198 -0
- package/README.md +61 -12
- package/affected_tests.py +1 -1
- package/atomic_io.py +8 -2
- package/code_graph.py +25 -8
- package/code_graph_go.py +526 -12
- package/code_graph_java.mjs +19 -10
- package/code_graph_js.cjs +9 -7
- package/comment_check.py +210 -0
- package/config.py +36 -4
- package/doc_check.py +120 -20
- package/doc_cli.py +239 -0
- package/doc_extras.py +561 -0
- package/doc_insights.py +300 -0
- package/doc_prose.py +371 -0
- package/doc_scope.py +869 -0
- package/doc_validate.py +717 -0
- package/duplicates.py +15 -7
- package/hook_decision.py +62 -14
- package/install.py +59 -1
- package/install_runtime.py +30 -4
- package/layers.py +237 -0
- package/locales/pt-BR.json +10 -1
- package/model_client.py +10 -5
- package/model_defaults.py +9 -6
- package/model_health.py +9 -3
- package/model_upgrade.py +1 -1
- package/package.json +1 -1
- package/pr_impact.py +84 -0
- package/project_insights_ui.py +1 -1
- package/project_maps_ui.py +1 -1
- package/redirect_rule.py +20 -8
- package/setup_ui.py +21 -6
- package/smart_tool_daemon.py +118 -18
- package/usage_meter.py +6 -1
- package/version.py +1 -1
- package/web_adapters/browser/prune_camoufox.py +66 -0
- package/web_fetch.py +1 -1
package/doc_scope.py
ADDED
|
@@ -0,0 +1,869 @@
|
|
|
1
|
+
"""Documentation scope of a project in three levels tied to the code, written to docs/scope in the project:
|
|
2
|
+
|
|
3
|
+
- technical/: a page per folder, a section per module and an entry per public function anchored on its symbol
|
|
4
|
+
(path::name), with signature, first doc line, layer, callers, callees and the tests that reach it, plus a Mermaid
|
|
5
|
+
flow of the folders grouped by declared layer;
|
|
6
|
+
- functional.md: one anchor per entry point (the functions of the highest declared layer, every main and the
|
|
7
|
+
functions a `__main__` block runs) with the functions its static call flow reaches, grouped by layer and linked to
|
|
8
|
+
their technical anchors, and the command lines the code declares with argparse;
|
|
9
|
+
- simple.md: the plain-language summary, one block per entry point linked to the functional anchor.
|
|
10
|
+
|
|
11
|
+
The skeleton comes from the code graph without a model. The prose (a paragraph per module, per entry point and the
|
|
12
|
+
simple README) is written between prose markers by the Docs model (doc_prose) or by hand and kept across
|
|
13
|
+
regenerations, as terraform-docs keeps the text around its generated block. Each anchor records the signature it was generated from; `check` and the edit
|
|
14
|
+
hook flag a document that no longer matches the code, like Swimm's smart tokens. The prose block of an entry point
|
|
15
|
+
records the code of the function it was written for (`code="..."`), so `check` and `validate` flag a paragraph left
|
|
16
|
+
behind by a body change that kept the signature. The link lives in the documents and in the `anchors` query, never
|
|
17
|
+
inside docstrings."""
|
|
18
|
+
import hashlib
|
|
19
|
+
import html
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import posixpath
|
|
23
|
+
import re
|
|
24
|
+
import tomllib
|
|
25
|
+
from collections import defaultdict, deque
|
|
26
|
+
|
|
27
|
+
import atomic_io
|
|
28
|
+
import code_graph
|
|
29
|
+
import config
|
|
30
|
+
import doc_check
|
|
31
|
+
import doc_cli
|
|
32
|
+
import index_profile
|
|
33
|
+
import index_scope
|
|
34
|
+
import layers
|
|
35
|
+
|
|
36
|
+
DOCS_DIR = "docs/scope"
|
|
37
|
+
GENERATED = "<!-- smart-tool:generated: regenerate with project_manage action=docs generate=true; write prose only between prose markers -->"
|
|
38
|
+
TEST_DEPTH = 3
|
|
39
|
+
FLOW_DEPTH = 6
|
|
40
|
+
MAX_FLOW = 60
|
|
41
|
+
MAX_LINKS = 12
|
|
42
|
+
MAX_VISITED = 400
|
|
43
|
+
MAX_FLOW_EDGES = 60
|
|
44
|
+
MAX_PAGE_ENTRIES = 150
|
|
45
|
+
MIN_ROOT_FLOW = 3
|
|
46
|
+
MAX_ROOTS = 40
|
|
47
|
+
PROSE = re.compile(r'<!-- smart-tool:prose (.+?)(?: code="[0-9a-f]{12}")? -->\n(.*?)<!-- /smart-tool:prose -->', re.S)
|
|
48
|
+
STAMPED_PROSE = re.compile(r'<!-- smart-tool:prose (.+?)(?: code="([0-9a-f]{12})")? -->\n(.*?)<!-- /smart-tool:prose -->', re.S)
|
|
49
|
+
MARKER = re.compile(r'<!-- smart-tool:symbol id="([^"]+)" signature="([0-9a-f]{12})" -->')
|
|
50
|
+
LINK = re.compile(r"\]\(<([^>#]*)#([^>]+)>\)")
|
|
51
|
+
ANCHOR = re.compile(r'<a id="([^"]+)"></a>')
|
|
52
|
+
_SPACE = re.compile(r"\s+")
|
|
53
|
+
TRUNCATED = ("> The code graph was cut by its analysis limits: some functions and calls are missing from this "
|
|
54
|
+
"document.\n")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def signature_hash(signature):
|
|
58
|
+
"""Short hash of a signature with whitespace removed, as stored next to each anchor.
|
|
59
|
+
|
|
60
|
+
Parameters: signature (str).
|
|
61
|
+
Returns: str of 12 hex digits.
|
|
62
|
+
"""
|
|
63
|
+
return hashlib.sha1(_SPACE.sub("", signature or "").encode("utf-8")).hexdigest()[:12]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def code_stamp(source, function):
|
|
67
|
+
"""Short hash of a function's lines with whitespace removed: the version of the code a prose block describes.
|
|
68
|
+
|
|
69
|
+
Parameters: source (str) text of the function's file; function (dict) with 'start' and 'end' lines.
|
|
70
|
+
Returns: str of 12 hex digits.
|
|
71
|
+
"""
|
|
72
|
+
return signature_hash("\n".join(source.split("\n")[function["start"] - 1:function["end"]]))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def prose_open(key, stamp=None):
|
|
76
|
+
"""Opening marker of the prose block `key`, with the code stamp it was written for when there is one."""
|
|
77
|
+
return f"<!-- smart-tool:prose {key}" + (f' code="{stamp}"' if stamp else "") + " -->"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def stale_prose(pages, current):
|
|
81
|
+
"""Written entry-point prose of functional.md and simple.md whose code stamp is not the code of its function now,
|
|
82
|
+
a block without a stamp included: the paragraph may describe behavior the function no longer has.
|
|
83
|
+
|
|
84
|
+
Parameters: pages (dict) relative file -> text; current (dict) anchor -> code_stamp of the function now (a function
|
|
85
|
+
gone is left out, its signature marker already reports it).
|
|
86
|
+
Returns: list of dicts with file, line and id.
|
|
87
|
+
"""
|
|
88
|
+
found = []
|
|
89
|
+
for rel in ("functional.md", "simple.md"):
|
|
90
|
+
text = pages.get(rel) or ""
|
|
91
|
+
for block in STAMPED_PROSE.finditer(text):
|
|
92
|
+
key, stamp, body = block.groups()
|
|
93
|
+
if body.strip() and key in current and stamp != current[key]:
|
|
94
|
+
found.append({"file": rel, "line": text[:block.start()].count("\n") + 1, "id": key})
|
|
95
|
+
return found
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def anchor_ids(functions, path):
|
|
99
|
+
"""Document anchor of each function of one file: path::name, with @n (n from 2) when the name repeats, so an
|
|
100
|
+
anchor survives lines inserted above it.
|
|
101
|
+
|
|
102
|
+
Parameters: functions (list of doc_check function dicts with 'name' and 'start'); path (str) project-relative.
|
|
103
|
+
Returns: dict start line -> anchor id.
|
|
104
|
+
"""
|
|
105
|
+
seen, ids = defaultdict(int), {}
|
|
106
|
+
for function in sorted(functions, key=lambda f: f["start"]):
|
|
107
|
+
seen[function["name"]] += 1
|
|
108
|
+
n = seen[function["name"]]
|
|
109
|
+
ids[function["start"]] = f"{path}::{function['name']}" + (f"@{n}" if n > 1 else "")
|
|
110
|
+
return ids
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _slug(folder):
|
|
114
|
+
return folder.replace("/", "--") if folder else "_root"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _pages(model, public):
|
|
118
|
+
"""Technical page of each public function: one page per folder, split into one page per module (in a subfolder
|
|
119
|
+
named after the folder's page) when the folder has more than MAX_PAGE_ENTRIES entries."""
|
|
120
|
+
by_folder = defaultdict(list)
|
|
121
|
+
for anchor in public:
|
|
122
|
+
by_folder[posixpath.dirname(model.function_of[anchor]["path"])].append(anchor)
|
|
123
|
+
page_of, titles = {}, {}
|
|
124
|
+
for folder, anchors in by_folder.items():
|
|
125
|
+
split = len(anchors) > MAX_PAGE_ENTRIES
|
|
126
|
+
for anchor in anchors:
|
|
127
|
+
path = model.function_of[anchor]["path"]
|
|
128
|
+
page = f"technical/{_slug(folder)}/{posixpath.basename(path)}.md" if split else f"technical/{_slug(folder)}.md"
|
|
129
|
+
page_of[anchor] = page
|
|
130
|
+
title = path if split else folder or "(project root)"
|
|
131
|
+
if titles.setdefault(page, title) != title:
|
|
132
|
+
raise ValueError(f"{titles[page]} and {title} map to the same page {page}; rename one of them.")
|
|
133
|
+
return page_of, titles
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _escape(text):
|
|
137
|
+
return html.escape(text, quote=True)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _link(text, page, anchor):
|
|
141
|
+
return f"[`{text}`](<{page}#{_escape(anchor)}>)"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def registered(function):
|
|
145
|
+
"""True when a decorator other than staticmethod or classmethod wraps the function (a property, a route, a
|
|
146
|
+
registration): something outside the call graph calls it."""
|
|
147
|
+
return any(d not in ("staticmethod", "classmethod") for d in function.get("decorators", ()))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _with_pending_notice(text):
|
|
151
|
+
"""The page with a notice under its navigation line while any prose block is empty; the notice sits outside the
|
|
152
|
+
markers, so it is never taken for prose and goes away once every block is written."""
|
|
153
|
+
pending = sum(1 for _key, body in PROSE.findall(text) if not body.strip())
|
|
154
|
+
if not pending:
|
|
155
|
+
return text
|
|
156
|
+
lines = text.split("\n")
|
|
157
|
+
at = next(i for i, line in enumerate(lines) if i and line.startswith("["))
|
|
158
|
+
notice = (f"> Prose pending: {pending} block(s) between the `smart-tool:prose` markers are empty. The text is not "
|
|
159
|
+
"generated with the skeleton: project_manage action=docs write_prose=true writes it with the Docs model, "
|
|
160
|
+
"checking each paragraph against the code, or write it by hand; regenerating keeps what is written and "
|
|
161
|
+
"removes this notice when none is left.")
|
|
162
|
+
return "\n".join(lines[:at + 1] + ["", notice] + lines[at + 1:])
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _prose(key, kept, stamp=None):
|
|
166
|
+
"""A prose block: written text keeps the stamp it was written under, and text without one (rewritten by hand, its
|
|
167
|
+
stamp deleted) or an empty block takes stamp, the code as it is now."""
|
|
168
|
+
body, written_for = kept.get(key, ("", None))
|
|
169
|
+
return f"{prose_open(key, (written_for or stamp) if body else stamp)}\n{body}<!-- /smart-tool:prose -->\n"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
class _Model:
|
|
173
|
+
"""Graph, layer rule and per-function facts of the project's current view, read once per request."""
|
|
174
|
+
|
|
175
|
+
def __init__(self, root, view_id):
|
|
176
|
+
self.root = root
|
|
177
|
+
self._texts = {}
|
|
178
|
+
self._entries = None
|
|
179
|
+
self._files = None
|
|
180
|
+
self._command_lines = None
|
|
181
|
+
self._flows = {}
|
|
182
|
+
self._runners = {}
|
|
183
|
+
self.roots_hidden = 0
|
|
184
|
+
self.graph = code_graph.build_current(root, view_id, shared=True)
|
|
185
|
+
self.scope = index_scope.load_scope(root) or {}
|
|
186
|
+
profile = self.profile = index_profile.current(self.scope.get("profile"))
|
|
187
|
+
self.rule = layers.load(root)
|
|
188
|
+
self.kind = {}
|
|
189
|
+
by_path = defaultdict(list)
|
|
190
|
+
for symbol in self.graph.get("symbols") or []:
|
|
191
|
+
self.kind.setdefault(symbol["path"], index_profile.kind(symbol["path"], profile))
|
|
192
|
+
by_path[symbol["path"]].append(symbol)
|
|
193
|
+
self.anchor_of, self.function_of, self.symbol_of = {}, {}, {}
|
|
194
|
+
for path, symbols in by_path.items():
|
|
195
|
+
if not doc_check.supported(path) or self.kind[path] not in ("code", "test"):
|
|
196
|
+
continue
|
|
197
|
+
source = self.read(path)
|
|
198
|
+
if source is None:
|
|
199
|
+
continue
|
|
200
|
+
found = [f for f in doc_check.functions(path, source, symbols) if f.get("kind", "function")
|
|
201
|
+
in ("function", "method")]
|
|
202
|
+
ids = anchor_ids(found, path)
|
|
203
|
+
starts = {s["start_line"]: s for s in symbols}
|
|
204
|
+
for function in found:
|
|
205
|
+
symbol = starts.get(function["start"])
|
|
206
|
+
if symbol:
|
|
207
|
+
anchor = ids[function["start"]]
|
|
208
|
+
self.anchor_of[symbol["id"]] = anchor
|
|
209
|
+
self.function_of[anchor] = {**function, "path": path, "id": anchor}
|
|
210
|
+
self.symbol_of[anchor] = symbol
|
|
211
|
+
self.callers, self.callees = defaultdict(set), defaultdict(set)
|
|
212
|
+
self.used_by_module = set()
|
|
213
|
+
for call in self.graph.get("calls") or []:
|
|
214
|
+
source, target = self.anchor_of.get(call["source"]), self.anchor_of.get(call["target"])
|
|
215
|
+
if source and target and source != target:
|
|
216
|
+
self.callers[target].add(source)
|
|
217
|
+
self.callees[source].add(target)
|
|
218
|
+
elif target and not source and self.kind.get(call["source"].rpartition("::")[0]) == "code":
|
|
219
|
+
self.used_by_module.add(target)
|
|
220
|
+
annotated = bool(self.rule) and any(layer["annotations"] for layer in self.rule.layers)
|
|
221
|
+
self.layer_of = {path: self.rule.assign(path, self.read(path) if annotated else None) if self.rule else None
|
|
222
|
+
for path in by_path}
|
|
223
|
+
|
|
224
|
+
def read(self, path):
|
|
225
|
+
"""Text of a project file, read once; None when it cannot be read as UTF-8."""
|
|
226
|
+
if path not in self._texts:
|
|
227
|
+
try:
|
|
228
|
+
with open(os.path.join(self.root, path), encoding="utf-8") as stream:
|
|
229
|
+
self._texts[path] = stream.read()
|
|
230
|
+
except (OSError, UnicodeDecodeError):
|
|
231
|
+
self._texts[path] = None
|
|
232
|
+
return self._texts[path]
|
|
233
|
+
|
|
234
|
+
def files(self):
|
|
235
|
+
"""Project-relative paths of the indexed files, resolved once; the files that hold symbols when the project has
|
|
236
|
+
no valid scope."""
|
|
237
|
+
if self._files is None:
|
|
238
|
+
found = (index_scope.resolve_included_files(self.root, self.scope) if index_scope.valid_scope(self.scope)
|
|
239
|
+
else sorted(self.kind))
|
|
240
|
+
self._files = [p for p in found if not p.startswith(DOCS_DIR + "/")]
|
|
241
|
+
return list(self._files)
|
|
242
|
+
|
|
243
|
+
def kind_of(self, path):
|
|
244
|
+
"""code, test or doc for a project file under the index profile; None when the profile has no class for it."""
|
|
245
|
+
return self.kind[path] if path in self.kind else index_profile.kind(path, self.profile)
|
|
246
|
+
|
|
247
|
+
def public(self):
|
|
248
|
+
"""Anchors of the public code functions (the language convention requires their docstring), by file."""
|
|
249
|
+
return [a for a, f in sorted(self.function_of.items(), key=lambda item: (item[1]["path"], item[1]["start"]))
|
|
250
|
+
if f["required"] and self.kind[f["path"]] == "code"]
|
|
251
|
+
|
|
252
|
+
def tests_of(self, anchor):
|
|
253
|
+
"""Test functions that reach anchor through static calls within TEST_DEPTH hops, nearest first."""
|
|
254
|
+
found, seen, frontier = [], {anchor}, [anchor]
|
|
255
|
+
for _hop in range(TEST_DEPTH):
|
|
256
|
+
reached = []
|
|
257
|
+
for target in frontier:
|
|
258
|
+
for caller in sorted(self.callers[target]):
|
|
259
|
+
if caller in seen or len(seen) > MAX_VISITED:
|
|
260
|
+
continue
|
|
261
|
+
seen.add(caller)
|
|
262
|
+
if self.kind[self.function_of[caller]["path"]] == "test":
|
|
263
|
+
found.append(caller)
|
|
264
|
+
else:
|
|
265
|
+
reached.append(caller)
|
|
266
|
+
frontier = reached
|
|
267
|
+
return found
|
|
268
|
+
|
|
269
|
+
def flow_edges(self, anchor):
|
|
270
|
+
"""Calls (caller, callee) that lead from anchor to the code functions within FLOW_DEPTH hops, one per reached
|
|
271
|
+
function, in visiting order."""
|
|
272
|
+
if anchor not in self._flows:
|
|
273
|
+
self._flows[anchor] = self._walk_flow(anchor)
|
|
274
|
+
return list(self._flows[anchor])
|
|
275
|
+
|
|
276
|
+
def _walk_flow(self, anchor):
|
|
277
|
+
seen, queue, edges = {anchor}, deque([(anchor, 0)]), []
|
|
278
|
+
while queue and len(edges) < MAX_FLOW:
|
|
279
|
+
current, depth = queue.popleft()
|
|
280
|
+
if depth == FLOW_DEPTH:
|
|
281
|
+
continue
|
|
282
|
+
for callee in sorted(self.callees[current]):
|
|
283
|
+
if callee not in seen and self.kind[self.function_of[callee]["path"]] == "code":
|
|
284
|
+
seen.add(callee)
|
|
285
|
+
edges.append((current, callee))
|
|
286
|
+
queue.append((callee, depth + 1))
|
|
287
|
+
return edges[:MAX_FLOW]
|
|
288
|
+
|
|
289
|
+
def flow(self, anchor):
|
|
290
|
+
"""Code functions reached from anchor through static calls within FLOW_DEPTH hops, in visiting order."""
|
|
291
|
+
return [callee for _caller, callee in self.flow_edges(anchor)]
|
|
292
|
+
|
|
293
|
+
def _python_code(self):
|
|
294
|
+
return [p for p in self.files() if p.endswith(".py") and self.kind_of(p) == "code"]
|
|
295
|
+
|
|
296
|
+
def launched(self):
|
|
297
|
+
"""Anchors of the functions a `__main__` block of a code file runs (the script's own main or the function a
|
|
298
|
+
launcher script imports and calls)."""
|
|
299
|
+
files, by_name, found = set(self.files()), {}, []
|
|
300
|
+
for anchor, function in sorted(self.function_of.items(), key=lambda i: (i[1]["path"], i[1]["start"])):
|
|
301
|
+
by_name.setdefault((function["path"], function["name"]), anchor)
|
|
302
|
+
for path in self._python_code():
|
|
303
|
+
source = self.read(path)
|
|
304
|
+
for pair in doc_cli.launched(path, source, files) if source else []:
|
|
305
|
+
if pair in by_name and by_name[pair] not in found:
|
|
306
|
+
found.append(by_name[pair])
|
|
307
|
+
return found
|
|
308
|
+
|
|
309
|
+
def command_lines(self):
|
|
310
|
+
"""argparse command lines of the code files (see doc_cli.definitions), by file and line, parsed once."""
|
|
311
|
+
if self._command_lines is None:
|
|
312
|
+
found = []
|
|
313
|
+
for path in self._python_code():
|
|
314
|
+
source = self.read(path)
|
|
315
|
+
found += doc_cli.definitions(path, source) if source else []
|
|
316
|
+
self._command_lines = found
|
|
317
|
+
return self._command_lines
|
|
318
|
+
|
|
319
|
+
def entry_points(self):
|
|
320
|
+
"""Entry points: every function named main, the functions a `__main__` block runs and, when layers are
|
|
321
|
+
declared, the public functions of the highest layer; without layers, the roots of the call graph (see roots)."""
|
|
322
|
+
if self._entries is None:
|
|
323
|
+
top = self.rule.layers[0]["name"] if self.rule else None
|
|
324
|
+
ordered = sorted(self.function_of.items(), key=lambda item: (item[1]["path"], item[1]["start"]))
|
|
325
|
+
code = [(a, f) for a, f in ordered if self.kind[f["path"]] == "code"]
|
|
326
|
+
chosen = {a for a, f in code if f["name"].split(".")[-1] == "main"
|
|
327
|
+
or top and f["required"] and self.layer_of[f["path"]] == top} | set(self.launched())
|
|
328
|
+
if not self.rule:
|
|
329
|
+
chosen |= set(self.roots(code))
|
|
330
|
+
self._entries = [a for a, _f in code if a in chosen]
|
|
331
|
+
return self._entries
|
|
332
|
+
|
|
333
|
+
def roots(self, code):
|
|
334
|
+
"""The features of a project without declared layers: public functions no code function calls whose flow
|
|
335
|
+
reaches at least MIN_ROOT_FLOW others (a command, a UI action, a job), the MAX_ROOTS with the widest flow."""
|
|
336
|
+
found = []
|
|
337
|
+
for anchor, function in code:
|
|
338
|
+
if not function["required"] or function["name"].split(".")[-1].startswith("__") or registered(function) \
|
|
339
|
+
or self.callers_in_code(anchor):
|
|
340
|
+
continue
|
|
341
|
+
reach = len(self.flow(anchor))
|
|
342
|
+
if reach >= MIN_ROOT_FLOW:
|
|
343
|
+
found.append((-reach, function["path"], function["start"], anchor))
|
|
344
|
+
found.sort()
|
|
345
|
+
self.roots_hidden = max(0, len(found) - MAX_ROOTS)
|
|
346
|
+
return [anchor for *_key, anchor in found[:MAX_ROOTS]]
|
|
347
|
+
|
|
348
|
+
def callers_in_code(self, anchor):
|
|
349
|
+
"""Callers of anchor that are code functions, not tests."""
|
|
350
|
+
return [c for c in self.callers[anchor] if self.kind[self.function_of[c]["path"]] == "code"]
|
|
351
|
+
|
|
352
|
+
def code_reached(self, anchor):
|
|
353
|
+
"""Code functions reached from anchor (a test) through static calls within TEST_DEPTH hops."""
|
|
354
|
+
seen, queue = {anchor}, deque([(anchor, 0)])
|
|
355
|
+
while queue:
|
|
356
|
+
current, depth = queue.popleft()
|
|
357
|
+
if depth == TEST_DEPTH:
|
|
358
|
+
continue
|
|
359
|
+
for callee in self.callees[current]:
|
|
360
|
+
if callee not in seen:
|
|
361
|
+
seen.add(callee)
|
|
362
|
+
queue.append((callee, depth + 1))
|
|
363
|
+
return [a for a in seen if self.kind[self.function_of[a]["path"]] == "code"]
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _kept_prose(root):
|
|
367
|
+
"""Prose blocks of the generated documents already in the project: (relative file, key) -> (text, code stamp or
|
|
368
|
+
None)."""
|
|
369
|
+
kept, folder = {}, os.path.join(root, DOCS_DIR)
|
|
370
|
+
for rel in _generated_files(root):
|
|
371
|
+
with open(os.path.join(folder, rel), encoding="utf-8") as stream:
|
|
372
|
+
text = stream.read()
|
|
373
|
+
for key, stamp, body in STAMPED_PROSE.findall(text):
|
|
374
|
+
if body.strip():
|
|
375
|
+
kept[(rel, key)] = (body if body.endswith("\n") else body + "\n", stamp or None)
|
|
376
|
+
return kept
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _generated_files(root):
|
|
380
|
+
"""Markdown files under docs/scope written by this generator (first line is the GENERATED marker)."""
|
|
381
|
+
folder, found = os.path.join(root, DOCS_DIR), []
|
|
382
|
+
for current, _dirs, names in os.walk(folder):
|
|
383
|
+
for name in names:
|
|
384
|
+
if name.endswith(".md"):
|
|
385
|
+
full = os.path.join(current, name)
|
|
386
|
+
with open(full, encoding="utf-8") as stream:
|
|
387
|
+
if stream.readline().rstrip("\n") == GENERATED:
|
|
388
|
+
found.append(os.path.relpath(full, folder).replace(os.sep, "/"))
|
|
389
|
+
return sorted(found)
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _links(model, anchors, page_of, from_page):
|
|
393
|
+
shown = []
|
|
394
|
+
for anchor in sorted(anchors)[:MAX_LINKS]:
|
|
395
|
+
name = model.function_of[anchor]["name"]
|
|
396
|
+
target = page_of.get(anchor)
|
|
397
|
+
shown.append(_link(name, posixpath.relpath(target, posixpath.dirname(from_page) or "."), anchor) if target
|
|
398
|
+
else f"`{name}` ({model.function_of[anchor]['path']})")
|
|
399
|
+
more = len(anchors) - MAX_LINKS
|
|
400
|
+
return ", ".join(shown) + (f" (+{more} more)" if more > 0 else "")
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _technical(model, kept):
|
|
404
|
+
"""Technical pages by folder and their index with the Mermaid flow by layer: relative file -> text."""
|
|
405
|
+
public = model.public()
|
|
406
|
+
page_of, titles = _pages(model, public)
|
|
407
|
+
truncated = TRUNCATED if model.graph.get("truncated") else ""
|
|
408
|
+
pages, by_page = {}, defaultdict(list)
|
|
409
|
+
for anchor in public:
|
|
410
|
+
by_page[page_of[anchor]].append(anchor)
|
|
411
|
+
for page, anchors in sorted(by_page.items()):
|
|
412
|
+
here = posixpath.dirname(page)
|
|
413
|
+
lines = [GENERATED, f"# `{titles[page]}`", "", f"[Index](<{posixpath.relpath('technical/README.md', here)}>) · "
|
|
414
|
+
f"[Functional](<{posixpath.relpath('functional.md', here)}>)", ""]
|
|
415
|
+
if truncated:
|
|
416
|
+
lines += [truncated]
|
|
417
|
+
own, modules = {k: v for (f, k), v in kept.items() if f == page}, defaultdict(list)
|
|
418
|
+
for anchor in anchors:
|
|
419
|
+
modules[model.function_of[anchor]["path"]].append(anchor)
|
|
420
|
+
for path, entries in modules.items():
|
|
421
|
+
layer = model.layer_of.get(path)
|
|
422
|
+
lines += [f"## `{path}`", ""] + ([f"Layer: {layer}", ""] if layer else [])
|
|
423
|
+
lines += [_prose(path, own).rstrip("\n"), ""]
|
|
424
|
+
for anchor in entries:
|
|
425
|
+
function = model.function_of[anchor]
|
|
426
|
+
tests = model.tests_of(anchor)
|
|
427
|
+
lines += [f'<a id="{_escape(anchor)}"></a>', f"### `{function['name']}`", "",
|
|
428
|
+
f'<!-- smart-tool:symbol id="{_escape(anchor)}" signature="{signature_hash(function["signature"])}" -->',
|
|
429
|
+
"", f"`{function['signature'] or function['name']}` (line {function['start']})", "",
|
|
430
|
+
function["doc"] or "*No docstring yet.*", ""]
|
|
431
|
+
callers = {c for c in model.callers[anchor] if model.kind[model.function_of[c]["path"]] == "code"}
|
|
432
|
+
if callers:
|
|
433
|
+
lines.append(f"- Called by: {_links(model, callers, page_of, page)}")
|
|
434
|
+
if model.callees[anchor]:
|
|
435
|
+
lines.append(f"- Calls: {_links(model, model.callees[anchor], page_of, page)}")
|
|
436
|
+
if tests:
|
|
437
|
+
lines.append("- Tests: " + ", ".join(f"`{model.function_of[t]['name']}` ({model.function_of[t]['path']})"
|
|
438
|
+
for t in tests[:MAX_LINKS]))
|
|
439
|
+
lines.append("")
|
|
440
|
+
pages[page] = "\n".join(lines).rstrip("\n") + "\n"
|
|
441
|
+
pages["technical/README.md"] = _technical_index(model, by_page, titles, truncated)
|
|
442
|
+
return pages, page_of
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _technical_index(model, by_page, titles, truncated):
|
|
446
|
+
lines = [GENERATED, "# Technical reference", "", "[Scope](../README.md) · [Functional](../functional.md)", ""]
|
|
447
|
+
if truncated:
|
|
448
|
+
lines += [truncated]
|
|
449
|
+
lines += ["## Flow by layer", ""] + _mermaid(model) + [""]
|
|
450
|
+
lines += ["## Pages", "", "| Folder or module | Public functions | Documented |", "|---|---|---|"]
|
|
451
|
+
for page, anchors in sorted(by_page.items()):
|
|
452
|
+
documented = sum(1 for a in anchors if model.function_of[a]["documented"])
|
|
453
|
+
lines.append(f"| [`{titles[page]}`](<{posixpath.relpath(page, 'technical')}>) | {len(anchors)} | {documented} |")
|
|
454
|
+
return "\n".join(lines) + "\n"
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _mermaid(model):
|
|
458
|
+
"""Mermaid flowchart of code folders (one subgraph per declared layer, in order) and the static calls between them,
|
|
459
|
+
the strongest MAX_FLOW_EDGES edges."""
|
|
460
|
+
folder_of = {a: posixpath.dirname(f["path"]) or "." for a, f in model.function_of.items()
|
|
461
|
+
if model.kind[f["path"]] == "code"}
|
|
462
|
+
weights = defaultdict(int)
|
|
463
|
+
for source, targets in model.callees.items():
|
|
464
|
+
for target in targets:
|
|
465
|
+
if source in folder_of and target in folder_of and folder_of[source] != folder_of[target]:
|
|
466
|
+
weights[(folder_of[source], folder_of[target])] += 1
|
|
467
|
+
edges = sorted(weights.items(), key=lambda item: (-item[1], item[0]))[:MAX_FLOW_EDGES]
|
|
468
|
+
folders = sorted({f for edge, _n in edges for f in edge} | set(folder_of.values()))
|
|
469
|
+
node = {folder: f"f{i}" for i, folder in enumerate(folders)}
|
|
470
|
+
layer_of_folder = {}
|
|
471
|
+
for anchor, folder in folder_of.items():
|
|
472
|
+
layer = model.layer_of.get(model.function_of[anchor]["path"])
|
|
473
|
+
if layer:
|
|
474
|
+
layer_of_folder.setdefault(folder, layer)
|
|
475
|
+
lines = ["```mermaid", "flowchart TD"]
|
|
476
|
+
for index, layer in enumerate(model.rule.layers if model.rule else []):
|
|
477
|
+
members = [f for f in folders if layer_of_folder.get(f) == layer["name"]]
|
|
478
|
+
if members:
|
|
479
|
+
lines.append(f' subgraph L{index}["{layer["name"]}"]')
|
|
480
|
+
lines += [f' {node[f]}["{f}"]' for f in members]
|
|
481
|
+
lines.append(" end")
|
|
482
|
+
lines += [f' {node[f]}["{f}"]' for f in folders if f not in layer_of_folder]
|
|
483
|
+
lines += [f" {node[a]} -->|{n}| {node[b]}" for (a, b), n in edges]
|
|
484
|
+
lines.append("```")
|
|
485
|
+
if not model.rule:
|
|
486
|
+
lines.append("\nNo layers declared yet: project_manage action=layers suggests them; once confirmed, this flow "
|
|
487
|
+
"groups the folders by layer.")
|
|
488
|
+
return lines
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def entry_title(model, anchor, entries):
|
|
492
|
+
"""Heading text of an entry point: its name, plus its file when another entry point has the same name."""
|
|
493
|
+
function = model.function_of[anchor]
|
|
494
|
+
twins = sum(model.function_of[a]["name"] == function["name"] for a in entries)
|
|
495
|
+
return f"`{function['name']}`" + (f" in `{function['path']}`" if twins > 1 else "")
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def entry_link(model, anchor, entries, page):
|
|
499
|
+
"""Link to an entry point's section of the functional overview, labelled with its heading."""
|
|
500
|
+
return f"[{entry_title(model, anchor, entries)}](<{page}#{_escape(anchor)}>)"
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def cli_anchor(definition):
|
|
504
|
+
"""Anchor of a command line in the functional overview."""
|
|
505
|
+
return f"cli:{definition['path']}::{definition['function'] or '<module>'}@{definition['line']}"
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def cli_title(definition):
|
|
509
|
+
"""Heading text of a command line: the function that declares it and its file."""
|
|
510
|
+
where = f"`{definition['function']}` in `{definition['path']}`" if definition["function"] else f"`{definition['path']}`"
|
|
511
|
+
return where + (" (options added to a parser it receives)" if definition["shared"] else "")
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def cli_runners(model, definition, entries):
|
|
515
|
+
"""Entry points that run a command line: the function declaring it, or one whose flow reaches it; for a module-level
|
|
516
|
+
parser, the entry points of its file."""
|
|
517
|
+
key = (definition["path"], definition["function"], definition["line"], tuple(entries))
|
|
518
|
+
if key not in model._runners:
|
|
519
|
+
if not definition["function"]:
|
|
520
|
+
model._runners[key] = [a for a in entries if model.function_of[a]["path"] == definition["path"]]
|
|
521
|
+
else:
|
|
522
|
+
declaring = {a for a, f in model.function_of.items() if f["path"] == definition["path"]
|
|
523
|
+
and f["name"] == definition["function"] and f["start"] <= definition["line"] <= f["end"]}
|
|
524
|
+
model._runners[key] = [a for a in entries if a in declaring or declaring.intersection(model.flow(a))]
|
|
525
|
+
return list(model._runners[key])
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _command_lines(model, entries):
|
|
529
|
+
definitions = model.command_lines()
|
|
530
|
+
if not definitions:
|
|
531
|
+
return []
|
|
532
|
+
lines = ["## Command line", "", "The commands, arguments and options the code declares with argparse, with their "
|
|
533
|
+
"defaults and help text as written in the code.", ""]
|
|
534
|
+
for definition in definitions:
|
|
535
|
+
runners = cli_runners(model, definition, entries)
|
|
536
|
+
lines += [f'<a id="{_escape(cli_anchor(definition))}"></a>', f"### {cli_title(definition)}", "",
|
|
537
|
+
f"Declared at line {definition['line']}. Run by: "
|
|
538
|
+
+ (", ".join(entry_link(model, a, entries, "") for a in runners) if runners
|
|
539
|
+
else "no entry point reaches it through static calls") + ".", ""]
|
|
540
|
+
lines += doc_cli.lines(definition) + [""]
|
|
541
|
+
return lines
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def project_name(model):
|
|
545
|
+
"""Name the project declares in a manifest (pyproject.toml, package.json, Cargo.toml, go.mod), the one nearest the
|
|
546
|
+
root first, without an npm scope; the folder name when no manifest declares one."""
|
|
547
|
+
manifests = sorted((p for p in model.files() if posixpath.basename(p) in ("pyproject.toml", "package.json",
|
|
548
|
+
"Cargo.toml", "go.mod")),
|
|
549
|
+
key=lambda p: (p.count("/"), p))
|
|
550
|
+
for path in manifests:
|
|
551
|
+
text = model.read(path)
|
|
552
|
+
if not text:
|
|
553
|
+
continue
|
|
554
|
+
base = posixpath.basename(path)
|
|
555
|
+
try:
|
|
556
|
+
if base == "package.json":
|
|
557
|
+
name = json.loads(text).get("name")
|
|
558
|
+
elif base == "go.mod":
|
|
559
|
+
found = re.search(r"^module\s+(\S+)", text, re.M)
|
|
560
|
+
name = found.group(1).rsplit("/", 1)[-1] if found else None
|
|
561
|
+
else:
|
|
562
|
+
data = tomllib.loads(text)
|
|
563
|
+
name = (data.get("project") or data.get("package") or {}).get("name") \
|
|
564
|
+
or data.get("tool", {}).get("poetry", {}).get("name")
|
|
565
|
+
except (ValueError, AttributeError):
|
|
566
|
+
continue
|
|
567
|
+
if isinstance(name, str) and name.strip():
|
|
568
|
+
return name.strip().rsplit("/", 1)[-1], path
|
|
569
|
+
return os.path.basename(os.path.normpath(model.root)), None
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def _functional(model, kept, page_of):
|
|
573
|
+
entries = model.entry_points()
|
|
574
|
+
lines = [GENERATED, "# Functional overview", "", "[Scope](README.md) · [Simple](simple.md) · "
|
|
575
|
+
"[Technical](technical/README.md)", ""]
|
|
576
|
+
if model.graph.get("truncated"):
|
|
577
|
+
lines += [TRUNCATED]
|
|
578
|
+
if not entries:
|
|
579
|
+
lines += ["No entry point found: declare the layers (project_manage action=layers) so the functions of the "
|
|
580
|
+
"highest layer become entry points.", ""]
|
|
581
|
+
elif not model.rule:
|
|
582
|
+
lines += ["Without declared layers the entry points are every `main`, the functions a `__main__` block runs "
|
|
583
|
+
"and the public functions no code function "
|
|
584
|
+
f"calls whose flow reaches {MIN_ROOT_FLOW} or more others"
|
|
585
|
+
+ (f" ({model.roots_hidden} more with a narrower flow are not listed)" if model.roots_hidden else "")
|
|
586
|
+
+ "; declare the layers (project_manage action=layers) to choose them by layer.", ""]
|
|
587
|
+
own = {k: v for (f, k), v in kept.items() if f == "functional.md"}
|
|
588
|
+
order = [l["name"] for l in model.rule.layers] if model.rule else []
|
|
589
|
+
for anchor in entries:
|
|
590
|
+
function = model.function_of[anchor]
|
|
591
|
+
lines += [f'<a id="{_escape(anchor)}"></a>', f"## {entry_title(model, anchor, entries)}", "",
|
|
592
|
+
f'<!-- smart-tool:symbol id="{_escape(anchor)}" signature="{signature_hash(function["signature"])}" -->',
|
|
593
|
+
"", f"Entry point in `{function['path']}` (line {function['start']})."
|
|
594
|
+
+ (f" Technical entry: {_link(function['name'], page_of[anchor], anchor)}." if anchor in page_of else ""),
|
|
595
|
+
"", _prose(anchor, own, code_stamp(model.read(function["path"]), function)).rstrip("\n"), ""]
|
|
596
|
+
groups = defaultdict(list)
|
|
597
|
+
for reached in model.flow(anchor):
|
|
598
|
+
groups[model.layer_of.get(model.function_of[reached]["path"]) or "other"].append(reached)
|
|
599
|
+
for layer in order + ["other"]:
|
|
600
|
+
if groups.get(layer):
|
|
601
|
+
lines.append(f"- {layer}: {_links(model, groups[layer], page_of, 'functional.md')}")
|
|
602
|
+
lines.append("")
|
|
603
|
+
lines += _command_lines(model, entries)
|
|
604
|
+
return "\n".join(lines).rstrip("\n") + "\n", entries
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def _simple(model, kept, entries, vocabulary=()):
|
|
608
|
+
own = {k: v for (f, k), v in kept.items() if f == "simple.md"}
|
|
609
|
+
lines = [GENERATED, f"# {project_name(model)[0]} in plain words", "", "[Scope](README.md) · [Functional](functional.md)", "",
|
|
610
|
+
_prose("overview", own).rstrip("\n"), ""]
|
|
611
|
+
if vocabulary:
|
|
612
|
+
lines += [f"Words this project uses: {', '.join(vocabulary)} ([glossary](<extras/glossary.md>)).", ""]
|
|
613
|
+
lines += ["## What it does", ""]
|
|
614
|
+
for anchor in entries:
|
|
615
|
+
function = model.function_of[anchor]
|
|
616
|
+
lines += [f"### {entry_title(model, anchor, entries)}", "",
|
|
617
|
+
_prose(anchor, own, code_stamp(model.read(function["path"]), function)).rstrip("\n"),
|
|
618
|
+
f"[Details](<functional.md#{_escape(anchor)}>)", ""]
|
|
619
|
+
return "\n".join(lines).rstrip("\n") + "\n"
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def _index(model, counts, extras=()):
|
|
623
|
+
lines = [GENERATED, "# Documentation scope", "",
|
|
624
|
+
"- [Simple](simple.md): what the project does, in plain words.",
|
|
625
|
+
"- [Functional](functional.md): one section per entry point and the functions its flow reaches.",
|
|
626
|
+
"- [Technical](technical/README.md): one entry per public function with callers, calls, tests and layer.",
|
|
627
|
+
"- [Validation](validation.md): every claim of these pages checked against the code, with the verdict."]
|
|
628
|
+
lines += [f"- [{title}](<{rel}>)" for title, rel in extras]
|
|
629
|
+
lines += ["", f"{counts['public']} public functions on {counts['pages']} technical pages; "
|
|
630
|
+
f"{counts['entry_points']} entry points; layers: {counts['layers']}.", ""]
|
|
631
|
+
if model.graph.get("truncated"):
|
|
632
|
+
lines.append(TRUNCATED)
|
|
633
|
+
return "\n".join(lines).rstrip("\n") + "\n"
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def generate(root, view_id=None, limit=30, model=None):
|
|
637
|
+
"""Write the three documents to docs/scope, keeping the prose already written between prose markers.
|
|
638
|
+
|
|
639
|
+
Parameters: root (str) project folder; view_id (str or None) index view; limit (int) pending prose blocks listed;
|
|
640
|
+
model (_Model or None) a snapshot of the current code to reuse, built here when None.
|
|
641
|
+
Returns: dict led by 'validation' (verdict, score, claim counts, report path and the failed checks of
|
|
642
|
+
doc_validate.run over the pages just written), then the files written and removed, counts, the prose blocks still
|
|
643
|
+
empty and those dropped because their module or entry point is gone.
|
|
644
|
+
"""
|
|
645
|
+
folder = os.path.join(root, DOCS_DIR)
|
|
646
|
+
owned = set(_generated_files(root)) if os.path.isdir(folder) else set()
|
|
647
|
+
kept = _kept_prose(root) if owned else {}
|
|
648
|
+
model = model or _Model(root, view_id)
|
|
649
|
+
if not model.graph.get("symbols"):
|
|
650
|
+
raise ValueError(f"The code graph of {root} is empty: index the project first (project_manage action=index), "
|
|
651
|
+
"then generate again; no file was written.")
|
|
652
|
+
files, page_of = _technical(model, kept)
|
|
653
|
+
files["functional.md"], entries = _functional(model, kept, page_of)
|
|
654
|
+
import doc_extras
|
|
655
|
+
import doc_insights
|
|
656
|
+
enabled = config.docs_extras()
|
|
657
|
+
files["simple.md"] = _simple(model, kept, entries, doc_insights.simple_terms(model) if enabled["glossary"] else ())
|
|
658
|
+
counts = {"public": len(page_of), "pages": len(files) - 3, "entry_points": len(entries),
|
|
659
|
+
"layers": ", ".join(l["name"] for l in model.rule.layers) if model.rule else "none declared"}
|
|
660
|
+
extras, block = doc_extras.build(model, entries, counts, enabled)
|
|
661
|
+
files["README.md"] = _index(model, counts, [(doc_extras.TITLES[k], doc_extras.FILES[k]) for k in config.DOCS_EXTRAS
|
|
662
|
+
if doc_extras.FILES.get(k) in extras])
|
|
663
|
+
files.update(extras)
|
|
664
|
+
for rel in files:
|
|
665
|
+
if rel in ("simple.md", "functional.md") or rel.startswith("technical/") and rel != "technical/README.md":
|
|
666
|
+
files[rel] = _with_pending_notice(files[rel])
|
|
667
|
+
for rel in files:
|
|
668
|
+
full = os.path.join(folder, rel)
|
|
669
|
+
if os.path.exists(full) and rel not in owned:
|
|
670
|
+
raise ValueError(f"{full} exists and was not written by Smart Tool; move it before generating.")
|
|
671
|
+
for rel, text in files.items():
|
|
672
|
+
full = os.path.join(folder, rel)
|
|
673
|
+
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
674
|
+
atomic_io.write_text(full, text)
|
|
675
|
+
import doc_validate
|
|
676
|
+
removed = sorted(owned - set(files) - {doc_validate.REPORT})
|
|
677
|
+
for rel in removed:
|
|
678
|
+
os.remove(os.path.join(folder, rel))
|
|
679
|
+
parent = os.path.dirname(os.path.join(folder, rel))
|
|
680
|
+
if parent != os.path.join(folder, "technical") and not os.listdir(parent):
|
|
681
|
+
os.rmdir(parent)
|
|
682
|
+
pending = [{"file": f"{DOCS_DIR}/{rel}", "block": key} for rel, text in sorted(files.items())
|
|
683
|
+
for key, body in PROSE.findall(text) if not body.strip()]
|
|
684
|
+
index_file = doc_extras.write_index(root, block)
|
|
685
|
+
model._texts.pop(index_file, None)
|
|
686
|
+
present = {(rel, key) for rel, text in files.items() for key, _body in PROSE.findall(text)}
|
|
687
|
+
dropped = sorted(f"{DOCS_DIR}/{rel} {key}" for rel, key in kept if (rel, key) not in present)
|
|
688
|
+
validation = doc_validate.run(model, entries, limit)
|
|
689
|
+
summary = {k: validation[k] for k in ("verdict", "score", "claims_checked", "claims_failed", "prose_pending", "report")}
|
|
690
|
+
summary["failed_checks"] = [c for c in validation["checks"] if c["status"] == "fail"]
|
|
691
|
+
return {"validation": summary, "folder": folder, "written": len(files), "removed": [f"{DOCS_DIR}/{r}" for r in removed], **counts,
|
|
692
|
+
"prose_pending_total": len(pending), "prose_pending": pending[:limit], "prose_dropped": dropped[:limit],
|
|
693
|
+
"truncated": bool(model.graph.get("truncated")),
|
|
694
|
+
"extras": sorted(f"{DOCS_DIR}/{rel}" for rel in extras), "index_file": index_file if block is not None else None,
|
|
695
|
+
"guidance": (f"Validation FAILED: {summary['claims_failed']} claim(s) do not hold; read {validation['report']} "
|
|
696
|
+
"and fix the generator or the code before trusting these pages. " if summary["verdict"] == "fail"
|
|
697
|
+
else "") + "write_prose=true writes the empty prose blocks with the Docs model of the setup, checking each "
|
|
698
|
+
"paragraph against the code; to write by hand, keep the markers: a paragraph per module "
|
|
699
|
+
"(what it is for) on the technical pages, a paragraph per entry point (the feature, in the "
|
|
700
|
+
"user's words) in functional.md, and simple.md without technical terms. Regenerate after code "
|
|
701
|
+
"changes; check=true lists anchors whose function changed or is gone; validate=true checks "
|
|
702
|
+
"every claim again, the prose included."}
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _scan(root):
|
|
706
|
+
"""Anchors, signature markers and links of the generated documents: (markers, anchors per file, links)."""
|
|
707
|
+
markers, anchors, links = [], defaultdict(set), []
|
|
708
|
+
folder = os.path.join(root, DOCS_DIR)
|
|
709
|
+
for rel in _generated_files(root):
|
|
710
|
+
with open(os.path.join(folder, rel), encoding="utf-8") as stream:
|
|
711
|
+
for number, line in enumerate(stream, 1):
|
|
712
|
+
for anchor, digest in MARKER.findall(line):
|
|
713
|
+
markers.append({"file": rel, "line": number, "id": html.unescape(anchor), "signature": digest})
|
|
714
|
+
anchors[rel].update(html.unescape(a) for a in ANCHOR.findall(line))
|
|
715
|
+
links += [{"file": rel, "line": number, "target": target, "anchor": html.unescape(anchor)}
|
|
716
|
+
for target, anchor in LINK.findall(line)]
|
|
717
|
+
return markers, anchors, links
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def _current_code(root, paths, symbols_by_path):
|
|
721
|
+
"""Signature hash and code stamp of the functions in paths as they are on disk now: two dicts by anchor id."""
|
|
722
|
+
current, stamps = {}, {}
|
|
723
|
+
for path in paths:
|
|
724
|
+
try:
|
|
725
|
+
with open(os.path.join(root, path), encoding="utf-8") as stream:
|
|
726
|
+
source = stream.read()
|
|
727
|
+
except (OSError, UnicodeDecodeError):
|
|
728
|
+
continue
|
|
729
|
+
found = [f for f in doc_check.functions(path, source, symbols_by_path.get(path))
|
|
730
|
+
if f.get("kind", "function") in ("function", "method")]
|
|
731
|
+
ids = anchor_ids(found, path)
|
|
732
|
+
current.update({ids[f["start"]]: signature_hash(f["signature"]) for f in found})
|
|
733
|
+
stamps.update({ids[f["start"]]: code_stamp(source, f) for f in found})
|
|
734
|
+
return current, stamps
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def check(root, view_id=None, limit=30):
|
|
738
|
+
"""Drift of the generated documents: anchors whose function is gone or changed its signature since generation, and
|
|
739
|
+
links whose target anchor does not exist.
|
|
740
|
+
|
|
741
|
+
Parameters: root (str) project folder; view_id (str or None) index view; limit (int) findings listed.
|
|
742
|
+
Returns: dict with 'ok', 'findings' (file, line, id, issue: missing | signature_changed | broken_link) and counts.
|
|
743
|
+
"""
|
|
744
|
+
if not os.path.isdir(os.path.join(root, DOCS_DIR)):
|
|
745
|
+
raise ValueError(f"No {DOCS_DIR} in the project yet: generate it with project_manage action=docs generate=true.")
|
|
746
|
+
markers, anchors, links = _scan(root)
|
|
747
|
+
graph = code_graph.build_current(root, view_id, shared=True)
|
|
748
|
+
by_path = defaultdict(list)
|
|
749
|
+
for symbol in graph.get("symbols") or []:
|
|
750
|
+
by_path[symbol["path"]].append(symbol)
|
|
751
|
+
paths = {m["id"].rpartition("::")[0] for m in markers}
|
|
752
|
+
current, stamps = _current_code(root, paths, by_path)
|
|
753
|
+
findings = []
|
|
754
|
+
for marker in markers:
|
|
755
|
+
now = current.get(marker["id"])
|
|
756
|
+
if now != marker["signature"]:
|
|
757
|
+
findings.append({"file": f"{DOCS_DIR}/{marker['file']}", "line": marker["line"], "id": marker["id"],
|
|
758
|
+
"issue": "missing" if now is None else "signature_changed"})
|
|
759
|
+
for link in links:
|
|
760
|
+
target = posixpath.normpath(posixpath.join(posixpath.dirname(link["file"]), link["target"])) \
|
|
761
|
+
if link["target"] else link["file"]
|
|
762
|
+
if link["anchor"] not in anchors.get(target, ()):
|
|
763
|
+
findings.append({"file": f"{DOCS_DIR}/{link['file']}", "line": link["line"], "id": link["anchor"],
|
|
764
|
+
"issue": "broken_link"})
|
|
765
|
+
import doc_extras
|
|
766
|
+
for link in doc_extras.index_links(root):
|
|
767
|
+
links.append(link)
|
|
768
|
+
if link["anchor"] not in anchors.get(link["target"], ()):
|
|
769
|
+
findings.append({"file": link["file"], "line": link["line"], "id": link["anchor"], "issue": "broken_link"})
|
|
770
|
+
stale = stale_prose(_read_pages(root, ("functional.md", "simple.md")), stamps)
|
|
771
|
+
findings += [{**item, "file": f"{DOCS_DIR}/{item['file']}", "issue": "prose_stale"} for item in stale]
|
|
772
|
+
guidance = []
|
|
773
|
+
if len(stale) < len(findings):
|
|
774
|
+
guidance.append("Regenerate with project_manage action=docs generate=true: the skeleton follows the code and "
|
|
775
|
+
"the prose is kept; review the prose of the entries listed.")
|
|
776
|
+
if stale:
|
|
777
|
+
guidance.append("prose_stale: the code of the entry changed after its paragraph was written. Empty the block and "
|
|
778
|
+
"run docs write_prose=true, or rewrite it by hand and delete its code=\"...\" stamp, then "
|
|
779
|
+
"regenerate so the block records the code it now describes.")
|
|
780
|
+
return {"ok": not findings, "anchors": len(markers), "links": len(links), "findings_total": len(findings),
|
|
781
|
+
"findings": findings[:limit],
|
|
782
|
+
"guidance": " ".join(guidance) if findings else
|
|
783
|
+
"Every anchor matches the code, every link resolves and every entry paragraph was written for "
|
|
784
|
+
"the code of its function."}
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def _read_pages(root, names):
|
|
788
|
+
"""Text of the named generated documents that exist: relative file -> text."""
|
|
789
|
+
pages = {}
|
|
790
|
+
for rel in names:
|
|
791
|
+
path = os.path.join(root, DOCS_DIR, rel)
|
|
792
|
+
if os.path.isfile(path):
|
|
793
|
+
with open(path, encoding="utf-8") as stream:
|
|
794
|
+
pages[rel] = stream.read()
|
|
795
|
+
return pages
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def anchors(root, symbol, view_id=None):
|
|
799
|
+
"""Where a function is documented: its anchors in the generated documents with the drift state of each.
|
|
800
|
+
|
|
801
|
+
Parameters: root (str) project folder; symbol (str) name, Class.method or path::name; view_id (str or None).
|
|
802
|
+
Returns: dict with 'anchors' (file, line, id, level, current) and guidance when the documents do not exist.
|
|
803
|
+
"""
|
|
804
|
+
if not isinstance(symbol, str) or not symbol.strip():
|
|
805
|
+
raise ValueError("Pass symbol: a function, Class.method or path::name.")
|
|
806
|
+
if not os.path.isdir(os.path.join(root, DOCS_DIR)):
|
|
807
|
+
return {"symbol": symbol, "anchors": [],
|
|
808
|
+
"guidance": f"No {DOCS_DIR} yet: generate it with project_manage action=docs generate=true."}
|
|
809
|
+
path, _sep, name = symbol.strip().rpartition("::")
|
|
810
|
+
path = path.replace("\\", "/").removeprefix("./")
|
|
811
|
+
markers, _anchors, _links = _scan(root)
|
|
812
|
+
|
|
813
|
+
def wanted(anchor):
|
|
814
|
+
where, _s, full = anchor.rpartition("::")
|
|
815
|
+
base = full.split("@")[0]
|
|
816
|
+
return (not path or where == path or where.endswith("/" + path)) and (base == name or base.split(".")[-1] == name)
|
|
817
|
+
|
|
818
|
+
found = [m for m in markers if wanted(m["id"])]
|
|
819
|
+
graph = code_graph.build_current(root, view_id, shared=True)
|
|
820
|
+
by_path = defaultdict(list)
|
|
821
|
+
for item in graph.get("symbols") or []:
|
|
822
|
+
by_path[item["path"]].append(item)
|
|
823
|
+
current, _stamps = _current_code(root, {m["id"].rpartition("::")[0] for m in found}, by_path)
|
|
824
|
+
return {"symbol": symbol.strip(),
|
|
825
|
+
"anchors": [{"file": f"{DOCS_DIR}/{m['file']}", "line": m["line"], "id": m["id"],
|
|
826
|
+
"level": "technical" if m["file"].startswith("technical/") else m["file"][:-3],
|
|
827
|
+
"current": current.get(m["id"]) == m["signature"]} for m in found]}
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
_MARKERS_CACHE = {}
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _markers_for(root, path):
|
|
834
|
+
"""Signature markers of path's functions in the generated documents, cached by the documents' modification times."""
|
|
835
|
+
folder = os.path.join(root, DOCS_DIR)
|
|
836
|
+
if not os.path.isdir(folder):
|
|
837
|
+
return []
|
|
838
|
+
files = _generated_files(root)
|
|
839
|
+
stamp = tuple((rel, os.stat(os.path.join(folder, rel)).st_mtime_ns) for rel in files)
|
|
840
|
+
cached = _MARKERS_CACHE.get(root)
|
|
841
|
+
if not cached or cached[0] != stamp:
|
|
842
|
+
cached = (stamp, _scan(root)[0])
|
|
843
|
+
_MARKERS_CACHE[root] = cached
|
|
844
|
+
return [m for m in cached[1] if m["id"].rpartition("::")[0] == path]
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def on_edit(root, rel, before, after_functions):
|
|
848
|
+
"""Anchors of the edited file that this edit makes stale (matched the code before, not after).
|
|
849
|
+
|
|
850
|
+
Parameters: root (str) project folder; rel (str) project-relative path; before (str or None) text before the edit;
|
|
851
|
+
after_functions (list) doc_check functions of the file after the edit.
|
|
852
|
+
Returns: list of {'file', 'id', 'issue'}.
|
|
853
|
+
"""
|
|
854
|
+
markers = _markers_for(root, rel)
|
|
855
|
+
if not markers:
|
|
856
|
+
return []
|
|
857
|
+
absolute = os.path.join(root, rel)
|
|
858
|
+
old = [f for f in doc_check.functions(absolute, before) if f.get("kind", "function") in ("function", "method")] \
|
|
859
|
+
if before else []
|
|
860
|
+
new = [f for f in after_functions if f.get("kind", "function") in ("function", "method")]
|
|
861
|
+
old_ids, new_ids = anchor_ids(old, rel), anchor_ids(new, rel)
|
|
862
|
+
then = {old_ids[f["start"]]: signature_hash(f["signature"]) for f in old}
|
|
863
|
+
now = {new_ids[f["start"]]: signature_hash(f["signature"]) for f in new}
|
|
864
|
+
stale = []
|
|
865
|
+
for marker in markers:
|
|
866
|
+
if then.get(marker["id"]) == marker["signature"] and now.get(marker["id"]) != marker["signature"]:
|
|
867
|
+
stale.append({"file": f"{DOCS_DIR}/{marker['file']}", "id": marker["id"],
|
|
868
|
+
"issue": "missing" if marker["id"] not in now else "signature_changed"})
|
|
869
|
+
return stale
|