@allansantos-dev/smart-tool 0.9.57 → 0.9.65

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/doc_scope.py ADDED
@@ -0,0 +1,869 @@
1
+ """Documentation scope of a project in three levels tied to the code, written to docs/scope in the project:
2
+
3
+ - technical/: a page per folder, a section per module and an entry per public function anchored on its symbol
4
+ (path::name), with signature, first doc line, layer, callers, callees and the tests that reach it, plus a Mermaid
5
+ flow of the folders grouped by declared layer;
6
+ - functional.md: one anchor per entry point (the functions of the highest declared layer, every main and the
7
+ functions a `__main__` block runs) with the functions its static call flow reaches, grouped by layer and linked to
8
+ their technical anchors, and the command lines the code declares with argparse;
9
+ - simple.md: the plain-language summary, one block per entry point linked to the functional anchor.
10
+
11
+ The skeleton comes from the code graph without a model. The prose (a paragraph per module, per entry point and the
12
+ simple README) is written between prose markers by the Docs model (doc_prose) or by hand and kept across
13
+ regenerations, as terraform-docs keeps the text around its generated block. Each anchor records the signature it was generated from; `check` and the edit
14
+ hook flag a document that no longer matches the code, like Swimm's smart tokens. The prose block of an entry point
15
+ records the code of the function it was written for (`code="..."`), so `check` and `validate` flag a paragraph left
16
+ behind by a body change that kept the signature. The link lives in the documents and in the `anchors` query, never
17
+ inside docstrings."""
18
+ import hashlib
19
+ import html
20
+ import json
21
+ import os
22
+ import posixpath
23
+ import re
24
+ import tomllib
25
+ from collections import defaultdict, deque
26
+
27
+ import atomic_io
28
+ import code_graph
29
+ import config
30
+ import doc_check
31
+ import doc_cli
32
+ import index_profile
33
+ import index_scope
34
+ import layers
35
+
36
+ DOCS_DIR = "docs/scope"
37
+ GENERATED = "<!-- smart-tool:generated: regenerate with project_manage action=docs generate=true; write prose only between prose markers -->"
38
+ TEST_DEPTH = 3
39
+ FLOW_DEPTH = 6
40
+ MAX_FLOW = 60
41
+ MAX_LINKS = 12
42
+ MAX_VISITED = 400
43
+ MAX_FLOW_EDGES = 60
44
+ MAX_PAGE_ENTRIES = 150
45
+ MIN_ROOT_FLOW = 3
46
+ MAX_ROOTS = 40
47
+ PROSE = re.compile(r'<!-- smart-tool:prose (.+?)(?: code="[0-9a-f]{12}")? -->\n(.*?)<!-- /smart-tool:prose -->', re.S)
48
+ STAMPED_PROSE = re.compile(r'<!-- smart-tool:prose (.+?)(?: code="([0-9a-f]{12})")? -->\n(.*?)<!-- /smart-tool:prose -->', re.S)
49
+ MARKER = re.compile(r'<!-- smart-tool:symbol id="([^"]+)" signature="([0-9a-f]{12})" -->')
50
+ LINK = re.compile(r"\]\(<([^>#]*)#([^>]+)>\)")
51
+ ANCHOR = re.compile(r'<a id="([^"]+)"></a>')
52
+ _SPACE = re.compile(r"\s+")
53
+ TRUNCATED = ("> The code graph was cut by its analysis limits: some functions and calls are missing from this "
54
+ "document.\n")
55
+
56
+
57
+ def signature_hash(signature):
58
+ """Short hash of a signature with whitespace removed, as stored next to each anchor.
59
+
60
+ Parameters: signature (str).
61
+ Returns: str of 12 hex digits.
62
+ """
63
+ return hashlib.sha1(_SPACE.sub("", signature or "").encode("utf-8")).hexdigest()[:12]
64
+
65
+
66
+ def code_stamp(source, function):
67
+ """Short hash of a function's lines with whitespace removed: the version of the code a prose block describes.
68
+
69
+ Parameters: source (str) text of the function's file; function (dict) with 'start' and 'end' lines.
70
+ Returns: str of 12 hex digits.
71
+ """
72
+ return signature_hash("\n".join(source.split("\n")[function["start"] - 1:function["end"]]))
73
+
74
+
75
+ def prose_open(key, stamp=None):
76
+ """Opening marker of the prose block `key`, with the code stamp it was written for when there is one."""
77
+ return f"<!-- smart-tool:prose {key}" + (f' code="{stamp}"' if stamp else "") + " -->"
78
+
79
+
80
+ def stale_prose(pages, current):
81
+ """Written entry-point prose of functional.md and simple.md whose code stamp is not the code of its function now,
82
+ a block without a stamp included: the paragraph may describe behavior the function no longer has.
83
+
84
+ Parameters: pages (dict) relative file -> text; current (dict) anchor -> code_stamp of the function now (a function
85
+ gone is left out, its signature marker already reports it).
86
+ Returns: list of dicts with file, line and id.
87
+ """
88
+ found = []
89
+ for rel in ("functional.md", "simple.md"):
90
+ text = pages.get(rel) or ""
91
+ for block in STAMPED_PROSE.finditer(text):
92
+ key, stamp, body = block.groups()
93
+ if body.strip() and key in current and stamp != current[key]:
94
+ found.append({"file": rel, "line": text[:block.start()].count("\n") + 1, "id": key})
95
+ return found
96
+
97
+
98
+ def anchor_ids(functions, path):
99
+ """Document anchor of each function of one file: path::name, with @n (n from 2) when the name repeats, so an
100
+ anchor survives lines inserted above it.
101
+
102
+ Parameters: functions (list of doc_check function dicts with 'name' and 'start'); path (str) project-relative.
103
+ Returns: dict start line -> anchor id.
104
+ """
105
+ seen, ids = defaultdict(int), {}
106
+ for function in sorted(functions, key=lambda f: f["start"]):
107
+ seen[function["name"]] += 1
108
+ n = seen[function["name"]]
109
+ ids[function["start"]] = f"{path}::{function['name']}" + (f"@{n}" if n > 1 else "")
110
+ return ids
111
+
112
+
113
+ def _slug(folder):
114
+ return folder.replace("/", "--") if folder else "_root"
115
+
116
+
117
+ def _pages(model, public):
118
+ """Technical page of each public function: one page per folder, split into one page per module (in a subfolder
119
+ named after the folder's page) when the folder has more than MAX_PAGE_ENTRIES entries."""
120
+ by_folder = defaultdict(list)
121
+ for anchor in public:
122
+ by_folder[posixpath.dirname(model.function_of[anchor]["path"])].append(anchor)
123
+ page_of, titles = {}, {}
124
+ for folder, anchors in by_folder.items():
125
+ split = len(anchors) > MAX_PAGE_ENTRIES
126
+ for anchor in anchors:
127
+ path = model.function_of[anchor]["path"]
128
+ page = f"technical/{_slug(folder)}/{posixpath.basename(path)}.md" if split else f"technical/{_slug(folder)}.md"
129
+ page_of[anchor] = page
130
+ title = path if split else folder or "(project root)"
131
+ if titles.setdefault(page, title) != title:
132
+ raise ValueError(f"{titles[page]} and {title} map to the same page {page}; rename one of them.")
133
+ return page_of, titles
134
+
135
+
136
+ def _escape(text):
137
+ return html.escape(text, quote=True)
138
+
139
+
140
+ def _link(text, page, anchor):
141
+ return f"[`{text}`](<{page}#{_escape(anchor)}>)"
142
+
143
+
144
+ def registered(function):
145
+ """True when a decorator other than staticmethod or classmethod wraps the function (a property, a route, a
146
+ registration): something outside the call graph calls it."""
147
+ return any(d not in ("staticmethod", "classmethod") for d in function.get("decorators", ()))
148
+
149
+
150
+ def _with_pending_notice(text):
151
+ """The page with a notice under its navigation line while any prose block is empty; the notice sits outside the
152
+ markers, so it is never taken for prose and goes away once every block is written."""
153
+ pending = sum(1 for _key, body in PROSE.findall(text) if not body.strip())
154
+ if not pending:
155
+ return text
156
+ lines = text.split("\n")
157
+ at = next(i for i, line in enumerate(lines) if i and line.startswith("["))
158
+ notice = (f"> Prose pending: {pending} block(s) between the `smart-tool:prose` markers are empty. The text is not "
159
+ "generated with the skeleton: project_manage action=docs write_prose=true writes it with the Docs model, "
160
+ "checking each paragraph against the code, or write it by hand; regenerating keeps what is written and "
161
+ "removes this notice when none is left.")
162
+ return "\n".join(lines[:at + 1] + ["", notice] + lines[at + 1:])
163
+
164
+
165
+ def _prose(key, kept, stamp=None):
166
+ """A prose block: written text keeps the stamp it was written under, and text without one (rewritten by hand, its
167
+ stamp deleted) or an empty block takes stamp, the code as it is now."""
168
+ body, written_for = kept.get(key, ("", None))
169
+ return f"{prose_open(key, (written_for or stamp) if body else stamp)}\n{body}<!-- /smart-tool:prose -->\n"
170
+
171
+
172
+ class _Model:
173
+ """Graph, layer rule and per-function facts of the project's current view, read once per request."""
174
+
175
+ def __init__(self, root, view_id):
176
+ self.root = root
177
+ self._texts = {}
178
+ self._entries = None
179
+ self._files = None
180
+ self._command_lines = None
181
+ self._flows = {}
182
+ self._runners = {}
183
+ self.roots_hidden = 0
184
+ self.graph = code_graph.build_current(root, view_id, shared=True)
185
+ self.scope = index_scope.load_scope(root) or {}
186
+ profile = self.profile = index_profile.current(self.scope.get("profile"))
187
+ self.rule = layers.load(root)
188
+ self.kind = {}
189
+ by_path = defaultdict(list)
190
+ for symbol in self.graph.get("symbols") or []:
191
+ self.kind.setdefault(symbol["path"], index_profile.kind(symbol["path"], profile))
192
+ by_path[symbol["path"]].append(symbol)
193
+ self.anchor_of, self.function_of, self.symbol_of = {}, {}, {}
194
+ for path, symbols in by_path.items():
195
+ if not doc_check.supported(path) or self.kind[path] not in ("code", "test"):
196
+ continue
197
+ source = self.read(path)
198
+ if source is None:
199
+ continue
200
+ found = [f for f in doc_check.functions(path, source, symbols) if f.get("kind", "function")
201
+ in ("function", "method")]
202
+ ids = anchor_ids(found, path)
203
+ starts = {s["start_line"]: s for s in symbols}
204
+ for function in found:
205
+ symbol = starts.get(function["start"])
206
+ if symbol:
207
+ anchor = ids[function["start"]]
208
+ self.anchor_of[symbol["id"]] = anchor
209
+ self.function_of[anchor] = {**function, "path": path, "id": anchor}
210
+ self.symbol_of[anchor] = symbol
211
+ self.callers, self.callees = defaultdict(set), defaultdict(set)
212
+ self.used_by_module = set()
213
+ for call in self.graph.get("calls") or []:
214
+ source, target = self.anchor_of.get(call["source"]), self.anchor_of.get(call["target"])
215
+ if source and target and source != target:
216
+ self.callers[target].add(source)
217
+ self.callees[source].add(target)
218
+ elif target and not source and self.kind.get(call["source"].rpartition("::")[0]) == "code":
219
+ self.used_by_module.add(target)
220
+ annotated = bool(self.rule) and any(layer["annotations"] for layer in self.rule.layers)
221
+ self.layer_of = {path: self.rule.assign(path, self.read(path) if annotated else None) if self.rule else None
222
+ for path in by_path}
223
+
224
+ def read(self, path):
225
+ """Text of a project file, read once; None when it cannot be read as UTF-8."""
226
+ if path not in self._texts:
227
+ try:
228
+ with open(os.path.join(self.root, path), encoding="utf-8") as stream:
229
+ self._texts[path] = stream.read()
230
+ except (OSError, UnicodeDecodeError):
231
+ self._texts[path] = None
232
+ return self._texts[path]
233
+
234
+ def files(self):
235
+ """Project-relative paths of the indexed files, resolved once; the files that hold symbols when the project has
236
+ no valid scope."""
237
+ if self._files is None:
238
+ found = (index_scope.resolve_included_files(self.root, self.scope) if index_scope.valid_scope(self.scope)
239
+ else sorted(self.kind))
240
+ self._files = [p for p in found if not p.startswith(DOCS_DIR + "/")]
241
+ return list(self._files)
242
+
243
+ def kind_of(self, path):
244
+ """code, test or doc for a project file under the index profile; None when the profile has no class for it."""
245
+ return self.kind[path] if path in self.kind else index_profile.kind(path, self.profile)
246
+
247
+ def public(self):
248
+ """Anchors of the public code functions (the language convention requires their docstring), by file."""
249
+ return [a for a, f in sorted(self.function_of.items(), key=lambda item: (item[1]["path"], item[1]["start"]))
250
+ if f["required"] and self.kind[f["path"]] == "code"]
251
+
252
+ def tests_of(self, anchor):
253
+ """Test functions that reach anchor through static calls within TEST_DEPTH hops, nearest first."""
254
+ found, seen, frontier = [], {anchor}, [anchor]
255
+ for _hop in range(TEST_DEPTH):
256
+ reached = []
257
+ for target in frontier:
258
+ for caller in sorted(self.callers[target]):
259
+ if caller in seen or len(seen) > MAX_VISITED:
260
+ continue
261
+ seen.add(caller)
262
+ if self.kind[self.function_of[caller]["path"]] == "test":
263
+ found.append(caller)
264
+ else:
265
+ reached.append(caller)
266
+ frontier = reached
267
+ return found
268
+
269
+ def flow_edges(self, anchor):
270
+ """Calls (caller, callee) that lead from anchor to the code functions within FLOW_DEPTH hops, one per reached
271
+ function, in visiting order."""
272
+ if anchor not in self._flows:
273
+ self._flows[anchor] = self._walk_flow(anchor)
274
+ return list(self._flows[anchor])
275
+
276
+ def _walk_flow(self, anchor):
277
+ seen, queue, edges = {anchor}, deque([(anchor, 0)]), []
278
+ while queue and len(edges) < MAX_FLOW:
279
+ current, depth = queue.popleft()
280
+ if depth == FLOW_DEPTH:
281
+ continue
282
+ for callee in sorted(self.callees[current]):
283
+ if callee not in seen and self.kind[self.function_of[callee]["path"]] == "code":
284
+ seen.add(callee)
285
+ edges.append((current, callee))
286
+ queue.append((callee, depth + 1))
287
+ return edges[:MAX_FLOW]
288
+
289
+ def flow(self, anchor):
290
+ """Code functions reached from anchor through static calls within FLOW_DEPTH hops, in visiting order."""
291
+ return [callee for _caller, callee in self.flow_edges(anchor)]
292
+
293
+ def _python_code(self):
294
+ return [p for p in self.files() if p.endswith(".py") and self.kind_of(p) == "code"]
295
+
296
+ def launched(self):
297
+ """Anchors of the functions a `__main__` block of a code file runs (the script's own main or the function a
298
+ launcher script imports and calls)."""
299
+ files, by_name, found = set(self.files()), {}, []
300
+ for anchor, function in sorted(self.function_of.items(), key=lambda i: (i[1]["path"], i[1]["start"])):
301
+ by_name.setdefault((function["path"], function["name"]), anchor)
302
+ for path in self._python_code():
303
+ source = self.read(path)
304
+ for pair in doc_cli.launched(path, source, files) if source else []:
305
+ if pair in by_name and by_name[pair] not in found:
306
+ found.append(by_name[pair])
307
+ return found
308
+
309
+ def command_lines(self):
310
+ """argparse command lines of the code files (see doc_cli.definitions), by file and line, parsed once."""
311
+ if self._command_lines is None:
312
+ found = []
313
+ for path in self._python_code():
314
+ source = self.read(path)
315
+ found += doc_cli.definitions(path, source) if source else []
316
+ self._command_lines = found
317
+ return self._command_lines
318
+
319
+ def entry_points(self):
320
+ """Entry points: every function named main, the functions a `__main__` block runs and, when layers are
321
+ declared, the public functions of the highest layer; without layers, the roots of the call graph (see roots)."""
322
+ if self._entries is None:
323
+ top = self.rule.layers[0]["name"] if self.rule else None
324
+ ordered = sorted(self.function_of.items(), key=lambda item: (item[1]["path"], item[1]["start"]))
325
+ code = [(a, f) for a, f in ordered if self.kind[f["path"]] == "code"]
326
+ chosen = {a for a, f in code if f["name"].split(".")[-1] == "main"
327
+ or top and f["required"] and self.layer_of[f["path"]] == top} | set(self.launched())
328
+ if not self.rule:
329
+ chosen |= set(self.roots(code))
330
+ self._entries = [a for a, _f in code if a in chosen]
331
+ return self._entries
332
+
333
+ def roots(self, code):
334
+ """The features of a project without declared layers: public functions no code function calls whose flow
335
+ reaches at least MIN_ROOT_FLOW others (a command, a UI action, a job), the MAX_ROOTS with the widest flow."""
336
+ found = []
337
+ for anchor, function in code:
338
+ if not function["required"] or function["name"].split(".")[-1].startswith("__") or registered(function) \
339
+ or self.callers_in_code(anchor):
340
+ continue
341
+ reach = len(self.flow(anchor))
342
+ if reach >= MIN_ROOT_FLOW:
343
+ found.append((-reach, function["path"], function["start"], anchor))
344
+ found.sort()
345
+ self.roots_hidden = max(0, len(found) - MAX_ROOTS)
346
+ return [anchor for *_key, anchor in found[:MAX_ROOTS]]
347
+
348
+ def callers_in_code(self, anchor):
349
+ """Callers of anchor that are code functions, not tests."""
350
+ return [c for c in self.callers[anchor] if self.kind[self.function_of[c]["path"]] == "code"]
351
+
352
+ def code_reached(self, anchor):
353
+ """Code functions reached from anchor (a test) through static calls within TEST_DEPTH hops."""
354
+ seen, queue = {anchor}, deque([(anchor, 0)])
355
+ while queue:
356
+ current, depth = queue.popleft()
357
+ if depth == TEST_DEPTH:
358
+ continue
359
+ for callee in self.callees[current]:
360
+ if callee not in seen:
361
+ seen.add(callee)
362
+ queue.append((callee, depth + 1))
363
+ return [a for a in seen if self.kind[self.function_of[a]["path"]] == "code"]
364
+
365
+
366
+ def _kept_prose(root):
367
+ """Prose blocks of the generated documents already in the project: (relative file, key) -> (text, code stamp or
368
+ None)."""
369
+ kept, folder = {}, os.path.join(root, DOCS_DIR)
370
+ for rel in _generated_files(root):
371
+ with open(os.path.join(folder, rel), encoding="utf-8") as stream:
372
+ text = stream.read()
373
+ for key, stamp, body in STAMPED_PROSE.findall(text):
374
+ if body.strip():
375
+ kept[(rel, key)] = (body if body.endswith("\n") else body + "\n", stamp or None)
376
+ return kept
377
+
378
+
379
+ def _generated_files(root):
380
+ """Markdown files under docs/scope written by this generator (first line is the GENERATED marker)."""
381
+ folder, found = os.path.join(root, DOCS_DIR), []
382
+ for current, _dirs, names in os.walk(folder):
383
+ for name in names:
384
+ if name.endswith(".md"):
385
+ full = os.path.join(current, name)
386
+ with open(full, encoding="utf-8") as stream:
387
+ if stream.readline().rstrip("\n") == GENERATED:
388
+ found.append(os.path.relpath(full, folder).replace(os.sep, "/"))
389
+ return sorted(found)
390
+
391
+
392
+ def _links(model, anchors, page_of, from_page):
393
+ shown = []
394
+ for anchor in sorted(anchors)[:MAX_LINKS]:
395
+ name = model.function_of[anchor]["name"]
396
+ target = page_of.get(anchor)
397
+ shown.append(_link(name, posixpath.relpath(target, posixpath.dirname(from_page) or "."), anchor) if target
398
+ else f"`{name}` ({model.function_of[anchor]['path']})")
399
+ more = len(anchors) - MAX_LINKS
400
+ return ", ".join(shown) + (f" (+{more} more)" if more > 0 else "")
401
+
402
+
403
+ def _technical(model, kept):
404
+ """Technical pages by folder and their index with the Mermaid flow by layer: relative file -> text."""
405
+ public = model.public()
406
+ page_of, titles = _pages(model, public)
407
+ truncated = TRUNCATED if model.graph.get("truncated") else ""
408
+ pages, by_page = {}, defaultdict(list)
409
+ for anchor in public:
410
+ by_page[page_of[anchor]].append(anchor)
411
+ for page, anchors in sorted(by_page.items()):
412
+ here = posixpath.dirname(page)
413
+ lines = [GENERATED, f"# `{titles[page]}`", "", f"[Index](<{posixpath.relpath('technical/README.md', here)}>) · "
414
+ f"[Functional](<{posixpath.relpath('functional.md', here)}>)", ""]
415
+ if truncated:
416
+ lines += [truncated]
417
+ own, modules = {k: v for (f, k), v in kept.items() if f == page}, defaultdict(list)
418
+ for anchor in anchors:
419
+ modules[model.function_of[anchor]["path"]].append(anchor)
420
+ for path, entries in modules.items():
421
+ layer = model.layer_of.get(path)
422
+ lines += [f"## `{path}`", ""] + ([f"Layer: {layer}", ""] if layer else [])
423
+ lines += [_prose(path, own).rstrip("\n"), ""]
424
+ for anchor in entries:
425
+ function = model.function_of[anchor]
426
+ tests = model.tests_of(anchor)
427
+ lines += [f'<a id="{_escape(anchor)}"></a>', f"### `{function['name']}`", "",
428
+ f'<!-- smart-tool:symbol id="{_escape(anchor)}" signature="{signature_hash(function["signature"])}" -->',
429
+ "", f"`{function['signature'] or function['name']}` (line {function['start']})", "",
430
+ function["doc"] or "*No docstring yet.*", ""]
431
+ callers = {c for c in model.callers[anchor] if model.kind[model.function_of[c]["path"]] == "code"}
432
+ if callers:
433
+ lines.append(f"- Called by: {_links(model, callers, page_of, page)}")
434
+ if model.callees[anchor]:
435
+ lines.append(f"- Calls: {_links(model, model.callees[anchor], page_of, page)}")
436
+ if tests:
437
+ lines.append("- Tests: " + ", ".join(f"`{model.function_of[t]['name']}` ({model.function_of[t]['path']})"
438
+ for t in tests[:MAX_LINKS]))
439
+ lines.append("")
440
+ pages[page] = "\n".join(lines).rstrip("\n") + "\n"
441
+ pages["technical/README.md"] = _technical_index(model, by_page, titles, truncated)
442
+ return pages, page_of
443
+
444
+
445
+ def _technical_index(model, by_page, titles, truncated):
446
+ lines = [GENERATED, "# Technical reference", "", "[Scope](../README.md) · [Functional](../functional.md)", ""]
447
+ if truncated:
448
+ lines += [truncated]
449
+ lines += ["## Flow by layer", ""] + _mermaid(model) + [""]
450
+ lines += ["## Pages", "", "| Folder or module | Public functions | Documented |", "|---|---|---|"]
451
+ for page, anchors in sorted(by_page.items()):
452
+ documented = sum(1 for a in anchors if model.function_of[a]["documented"])
453
+ lines.append(f"| [`{titles[page]}`](<{posixpath.relpath(page, 'technical')}>) | {len(anchors)} | {documented} |")
454
+ return "\n".join(lines) + "\n"
455
+
456
+
457
+ def _mermaid(model):
458
+ """Mermaid flowchart of code folders (one subgraph per declared layer, in order) and the static calls between them,
459
+ the strongest MAX_FLOW_EDGES edges."""
460
+ folder_of = {a: posixpath.dirname(f["path"]) or "." for a, f in model.function_of.items()
461
+ if model.kind[f["path"]] == "code"}
462
+ weights = defaultdict(int)
463
+ for source, targets in model.callees.items():
464
+ for target in targets:
465
+ if source in folder_of and target in folder_of and folder_of[source] != folder_of[target]:
466
+ weights[(folder_of[source], folder_of[target])] += 1
467
+ edges = sorted(weights.items(), key=lambda item: (-item[1], item[0]))[:MAX_FLOW_EDGES]
468
+ folders = sorted({f for edge, _n in edges for f in edge} | set(folder_of.values()))
469
+ node = {folder: f"f{i}" for i, folder in enumerate(folders)}
470
+ layer_of_folder = {}
471
+ for anchor, folder in folder_of.items():
472
+ layer = model.layer_of.get(model.function_of[anchor]["path"])
473
+ if layer:
474
+ layer_of_folder.setdefault(folder, layer)
475
+ lines = ["```mermaid", "flowchart TD"]
476
+ for index, layer in enumerate(model.rule.layers if model.rule else []):
477
+ members = [f for f in folders if layer_of_folder.get(f) == layer["name"]]
478
+ if members:
479
+ lines.append(f' subgraph L{index}["{layer["name"]}"]')
480
+ lines += [f' {node[f]}["{f}"]' for f in members]
481
+ lines.append(" end")
482
+ lines += [f' {node[f]}["{f}"]' for f in folders if f not in layer_of_folder]
483
+ lines += [f" {node[a]} -->|{n}| {node[b]}" for (a, b), n in edges]
484
+ lines.append("```")
485
+ if not model.rule:
486
+ lines.append("\nNo layers declared yet: project_manage action=layers suggests them; once confirmed, this flow "
487
+ "groups the folders by layer.")
488
+ return lines
489
+
490
+
491
+ def entry_title(model, anchor, entries):
492
+ """Heading text of an entry point: its name, plus its file when another entry point has the same name."""
493
+ function = model.function_of[anchor]
494
+ twins = sum(model.function_of[a]["name"] == function["name"] for a in entries)
495
+ return f"`{function['name']}`" + (f" in `{function['path']}`" if twins > 1 else "")
496
+
497
+
498
+ def entry_link(model, anchor, entries, page):
499
+ """Link to an entry point's section of the functional overview, labelled with its heading."""
500
+ return f"[{entry_title(model, anchor, entries)}](<{page}#{_escape(anchor)}>)"
501
+
502
+
503
+ def cli_anchor(definition):
504
+ """Anchor of a command line in the functional overview."""
505
+ return f"cli:{definition['path']}::{definition['function'] or '<module>'}@{definition['line']}"
506
+
507
+
508
+ def cli_title(definition):
509
+ """Heading text of a command line: the function that declares it and its file."""
510
+ where = f"`{definition['function']}` in `{definition['path']}`" if definition["function"] else f"`{definition['path']}`"
511
+ return where + (" (options added to a parser it receives)" if definition["shared"] else "")
512
+
513
+
514
+ def cli_runners(model, definition, entries):
515
+ """Entry points that run a command line: the function declaring it, or one whose flow reaches it; for a module-level
516
+ parser, the entry points of its file."""
517
+ key = (definition["path"], definition["function"], definition["line"], tuple(entries))
518
+ if key not in model._runners:
519
+ if not definition["function"]:
520
+ model._runners[key] = [a for a in entries if model.function_of[a]["path"] == definition["path"]]
521
+ else:
522
+ declaring = {a for a, f in model.function_of.items() if f["path"] == definition["path"]
523
+ and f["name"] == definition["function"] and f["start"] <= definition["line"] <= f["end"]}
524
+ model._runners[key] = [a for a in entries if a in declaring or declaring.intersection(model.flow(a))]
525
+ return list(model._runners[key])
526
+
527
+
528
+ def _command_lines(model, entries):
529
+ definitions = model.command_lines()
530
+ if not definitions:
531
+ return []
532
+ lines = ["## Command line", "", "The commands, arguments and options the code declares with argparse, with their "
533
+ "defaults and help text as written in the code.", ""]
534
+ for definition in definitions:
535
+ runners = cli_runners(model, definition, entries)
536
+ lines += [f'<a id="{_escape(cli_anchor(definition))}"></a>', f"### {cli_title(definition)}", "",
537
+ f"Declared at line {definition['line']}. Run by: "
538
+ + (", ".join(entry_link(model, a, entries, "") for a in runners) if runners
539
+ else "no entry point reaches it through static calls") + ".", ""]
540
+ lines += doc_cli.lines(definition) + [""]
541
+ return lines
542
+
543
+
544
+ def project_name(model):
545
+ """Name the project declares in a manifest (pyproject.toml, package.json, Cargo.toml, go.mod), the one nearest the
546
+ root first, without an npm scope; the folder name when no manifest declares one."""
547
+ manifests = sorted((p for p in model.files() if posixpath.basename(p) in ("pyproject.toml", "package.json",
548
+ "Cargo.toml", "go.mod")),
549
+ key=lambda p: (p.count("/"), p))
550
+ for path in manifests:
551
+ text = model.read(path)
552
+ if not text:
553
+ continue
554
+ base = posixpath.basename(path)
555
+ try:
556
+ if base == "package.json":
557
+ name = json.loads(text).get("name")
558
+ elif base == "go.mod":
559
+ found = re.search(r"^module\s+(\S+)", text, re.M)
560
+ name = found.group(1).rsplit("/", 1)[-1] if found else None
561
+ else:
562
+ data = tomllib.loads(text)
563
+ name = (data.get("project") or data.get("package") or {}).get("name") \
564
+ or data.get("tool", {}).get("poetry", {}).get("name")
565
+ except (ValueError, AttributeError):
566
+ continue
567
+ if isinstance(name, str) and name.strip():
568
+ return name.strip().rsplit("/", 1)[-1], path
569
+ return os.path.basename(os.path.normpath(model.root)), None
570
+
571
+
572
+ def _functional(model, kept, page_of):
573
+ entries = model.entry_points()
574
+ lines = [GENERATED, "# Functional overview", "", "[Scope](README.md) · [Simple](simple.md) · "
575
+ "[Technical](technical/README.md)", ""]
576
+ if model.graph.get("truncated"):
577
+ lines += [TRUNCATED]
578
+ if not entries:
579
+ lines += ["No entry point found: declare the layers (project_manage action=layers) so the functions of the "
580
+ "highest layer become entry points.", ""]
581
+ elif not model.rule:
582
+ lines += ["Without declared layers the entry points are every `main`, the functions a `__main__` block runs "
583
+ "and the public functions no code function "
584
+ f"calls whose flow reaches {MIN_ROOT_FLOW} or more others"
585
+ + (f" ({model.roots_hidden} more with a narrower flow are not listed)" if model.roots_hidden else "")
586
+ + "; declare the layers (project_manage action=layers) to choose them by layer.", ""]
587
+ own = {k: v for (f, k), v in kept.items() if f == "functional.md"}
588
+ order = [l["name"] for l in model.rule.layers] if model.rule else []
589
+ for anchor in entries:
590
+ function = model.function_of[anchor]
591
+ lines += [f'<a id="{_escape(anchor)}"></a>', f"## {entry_title(model, anchor, entries)}", "",
592
+ f'<!-- smart-tool:symbol id="{_escape(anchor)}" signature="{signature_hash(function["signature"])}" -->',
593
+ "", f"Entry point in `{function['path']}` (line {function['start']})."
594
+ + (f" Technical entry: {_link(function['name'], page_of[anchor], anchor)}." if anchor in page_of else ""),
595
+ "", _prose(anchor, own, code_stamp(model.read(function["path"]), function)).rstrip("\n"), ""]
596
+ groups = defaultdict(list)
597
+ for reached in model.flow(anchor):
598
+ groups[model.layer_of.get(model.function_of[reached]["path"]) or "other"].append(reached)
599
+ for layer in order + ["other"]:
600
+ if groups.get(layer):
601
+ lines.append(f"- {layer}: {_links(model, groups[layer], page_of, 'functional.md')}")
602
+ lines.append("")
603
+ lines += _command_lines(model, entries)
604
+ return "\n".join(lines).rstrip("\n") + "\n", entries
605
+
606
+
607
+ def _simple(model, kept, entries, vocabulary=()):
608
+ own = {k: v for (f, k), v in kept.items() if f == "simple.md"}
609
+ lines = [GENERATED, f"# {project_name(model)[0]} in plain words", "", "[Scope](README.md) · [Functional](functional.md)", "",
610
+ _prose("overview", own).rstrip("\n"), ""]
611
+ if vocabulary:
612
+ lines += [f"Words this project uses: {', '.join(vocabulary)} ([glossary](<extras/glossary.md>)).", ""]
613
+ lines += ["## What it does", ""]
614
+ for anchor in entries:
615
+ function = model.function_of[anchor]
616
+ lines += [f"### {entry_title(model, anchor, entries)}", "",
617
+ _prose(anchor, own, code_stamp(model.read(function["path"]), function)).rstrip("\n"),
618
+ f"[Details](<functional.md#{_escape(anchor)}>)", ""]
619
+ return "\n".join(lines).rstrip("\n") + "\n"
620
+
621
+
622
+ def _index(model, counts, extras=()):
623
+ lines = [GENERATED, "# Documentation scope", "",
624
+ "- [Simple](simple.md): what the project does, in plain words.",
625
+ "- [Functional](functional.md): one section per entry point and the functions its flow reaches.",
626
+ "- [Technical](technical/README.md): one entry per public function with callers, calls, tests and layer.",
627
+ "- [Validation](validation.md): every claim of these pages checked against the code, with the verdict."]
628
+ lines += [f"- [{title}](<{rel}>)" for title, rel in extras]
629
+ lines += ["", f"{counts['public']} public functions on {counts['pages']} technical pages; "
630
+ f"{counts['entry_points']} entry points; layers: {counts['layers']}.", ""]
631
+ if model.graph.get("truncated"):
632
+ lines.append(TRUNCATED)
633
+ return "\n".join(lines).rstrip("\n") + "\n"
634
+
635
+
636
+ def generate(root, view_id=None, limit=30, model=None):
637
+ """Write the three documents to docs/scope, keeping the prose already written between prose markers.
638
+
639
+ Parameters: root (str) project folder; view_id (str or None) index view; limit (int) pending prose blocks listed;
640
+ model (_Model or None) a snapshot of the current code to reuse, built here when None.
641
+ Returns: dict led by 'validation' (verdict, score, claim counts, report path and the failed checks of
642
+ doc_validate.run over the pages just written), then the files written and removed, counts, the prose blocks still
643
+ empty and those dropped because their module or entry point is gone.
644
+ """
645
+ folder = os.path.join(root, DOCS_DIR)
646
+ owned = set(_generated_files(root)) if os.path.isdir(folder) else set()
647
+ kept = _kept_prose(root) if owned else {}
648
+ model = model or _Model(root, view_id)
649
+ if not model.graph.get("symbols"):
650
+ raise ValueError(f"The code graph of {root} is empty: index the project first (project_manage action=index), "
651
+ "then generate again; no file was written.")
652
+ files, page_of = _technical(model, kept)
653
+ files["functional.md"], entries = _functional(model, kept, page_of)
654
+ import doc_extras
655
+ import doc_insights
656
+ enabled = config.docs_extras()
657
+ files["simple.md"] = _simple(model, kept, entries, doc_insights.simple_terms(model) if enabled["glossary"] else ())
658
+ counts = {"public": len(page_of), "pages": len(files) - 3, "entry_points": len(entries),
659
+ "layers": ", ".join(l["name"] for l in model.rule.layers) if model.rule else "none declared"}
660
+ extras, block = doc_extras.build(model, entries, counts, enabled)
661
+ files["README.md"] = _index(model, counts, [(doc_extras.TITLES[k], doc_extras.FILES[k]) for k in config.DOCS_EXTRAS
662
+ if doc_extras.FILES.get(k) in extras])
663
+ files.update(extras)
664
+ for rel in files:
665
+ if rel in ("simple.md", "functional.md") or rel.startswith("technical/") and rel != "technical/README.md":
666
+ files[rel] = _with_pending_notice(files[rel])
667
+ for rel in files:
668
+ full = os.path.join(folder, rel)
669
+ if os.path.exists(full) and rel not in owned:
670
+ raise ValueError(f"{full} exists and was not written by Smart Tool; move it before generating.")
671
+ for rel, text in files.items():
672
+ full = os.path.join(folder, rel)
673
+ os.makedirs(os.path.dirname(full), exist_ok=True)
674
+ atomic_io.write_text(full, text)
675
+ import doc_validate
676
+ removed = sorted(owned - set(files) - {doc_validate.REPORT})
677
+ for rel in removed:
678
+ os.remove(os.path.join(folder, rel))
679
+ parent = os.path.dirname(os.path.join(folder, rel))
680
+ if parent != os.path.join(folder, "technical") and not os.listdir(parent):
681
+ os.rmdir(parent)
682
+ pending = [{"file": f"{DOCS_DIR}/{rel}", "block": key} for rel, text in sorted(files.items())
683
+ for key, body in PROSE.findall(text) if not body.strip()]
684
+ index_file = doc_extras.write_index(root, block)
685
+ model._texts.pop(index_file, None)
686
+ present = {(rel, key) for rel, text in files.items() for key, _body in PROSE.findall(text)}
687
+ dropped = sorted(f"{DOCS_DIR}/{rel} {key}" for rel, key in kept if (rel, key) not in present)
688
+ validation = doc_validate.run(model, entries, limit)
689
+ summary = {k: validation[k] for k in ("verdict", "score", "claims_checked", "claims_failed", "prose_pending", "report")}
690
+ summary["failed_checks"] = [c for c in validation["checks"] if c["status"] == "fail"]
691
+ return {"validation": summary, "folder": folder, "written": len(files), "removed": [f"{DOCS_DIR}/{r}" for r in removed], **counts,
692
+ "prose_pending_total": len(pending), "prose_pending": pending[:limit], "prose_dropped": dropped[:limit],
693
+ "truncated": bool(model.graph.get("truncated")),
694
+ "extras": sorted(f"{DOCS_DIR}/{rel}" for rel in extras), "index_file": index_file if block is not None else None,
695
+ "guidance": (f"Validation FAILED: {summary['claims_failed']} claim(s) do not hold; read {validation['report']} "
696
+ "and fix the generator or the code before trusting these pages. " if summary["verdict"] == "fail"
697
+ else "") + "write_prose=true writes the empty prose blocks with the Docs model of the setup, checking each "
698
+ "paragraph against the code; to write by hand, keep the markers: a paragraph per module "
699
+ "(what it is for) on the technical pages, a paragraph per entry point (the feature, in the "
700
+ "user's words) in functional.md, and simple.md without technical terms. Regenerate after code "
701
+ "changes; check=true lists anchors whose function changed or is gone; validate=true checks "
702
+ "every claim again, the prose included."}
703
+
704
+
705
+ def _scan(root):
706
+ """Anchors, signature markers and links of the generated documents: (markers, anchors per file, links)."""
707
+ markers, anchors, links = [], defaultdict(set), []
708
+ folder = os.path.join(root, DOCS_DIR)
709
+ for rel in _generated_files(root):
710
+ with open(os.path.join(folder, rel), encoding="utf-8") as stream:
711
+ for number, line in enumerate(stream, 1):
712
+ for anchor, digest in MARKER.findall(line):
713
+ markers.append({"file": rel, "line": number, "id": html.unescape(anchor), "signature": digest})
714
+ anchors[rel].update(html.unescape(a) for a in ANCHOR.findall(line))
715
+ links += [{"file": rel, "line": number, "target": target, "anchor": html.unescape(anchor)}
716
+ for target, anchor in LINK.findall(line)]
717
+ return markers, anchors, links
718
+
719
+
720
+ def _current_code(root, paths, symbols_by_path):
721
+ """Signature hash and code stamp of the functions in paths as they are on disk now: two dicts by anchor id."""
722
+ current, stamps = {}, {}
723
+ for path in paths:
724
+ try:
725
+ with open(os.path.join(root, path), encoding="utf-8") as stream:
726
+ source = stream.read()
727
+ except (OSError, UnicodeDecodeError):
728
+ continue
729
+ found = [f for f in doc_check.functions(path, source, symbols_by_path.get(path))
730
+ if f.get("kind", "function") in ("function", "method")]
731
+ ids = anchor_ids(found, path)
732
+ current.update({ids[f["start"]]: signature_hash(f["signature"]) for f in found})
733
+ stamps.update({ids[f["start"]]: code_stamp(source, f) for f in found})
734
+ return current, stamps
735
+
736
+
737
+ def check(root, view_id=None, limit=30):
738
+ """Drift of the generated documents: anchors whose function is gone or changed its signature since generation, and
739
+ links whose target anchor does not exist.
740
+
741
+ Parameters: root (str) project folder; view_id (str or None) index view; limit (int) findings listed.
742
+ Returns: dict with 'ok', 'findings' (file, line, id, issue: missing | signature_changed | broken_link) and counts.
743
+ """
744
+ if not os.path.isdir(os.path.join(root, DOCS_DIR)):
745
+ raise ValueError(f"No {DOCS_DIR} in the project yet: generate it with project_manage action=docs generate=true.")
746
+ markers, anchors, links = _scan(root)
747
+ graph = code_graph.build_current(root, view_id, shared=True)
748
+ by_path = defaultdict(list)
749
+ for symbol in graph.get("symbols") or []:
750
+ by_path[symbol["path"]].append(symbol)
751
+ paths = {m["id"].rpartition("::")[0] for m in markers}
752
+ current, stamps = _current_code(root, paths, by_path)
753
+ findings = []
754
+ for marker in markers:
755
+ now = current.get(marker["id"])
756
+ if now != marker["signature"]:
757
+ findings.append({"file": f"{DOCS_DIR}/{marker['file']}", "line": marker["line"], "id": marker["id"],
758
+ "issue": "missing" if now is None else "signature_changed"})
759
+ for link in links:
760
+ target = posixpath.normpath(posixpath.join(posixpath.dirname(link["file"]), link["target"])) \
761
+ if link["target"] else link["file"]
762
+ if link["anchor"] not in anchors.get(target, ()):
763
+ findings.append({"file": f"{DOCS_DIR}/{link['file']}", "line": link["line"], "id": link["anchor"],
764
+ "issue": "broken_link"})
765
+ import doc_extras
766
+ for link in doc_extras.index_links(root):
767
+ links.append(link)
768
+ if link["anchor"] not in anchors.get(link["target"], ()):
769
+ findings.append({"file": link["file"], "line": link["line"], "id": link["anchor"], "issue": "broken_link"})
770
+ stale = stale_prose(_read_pages(root, ("functional.md", "simple.md")), stamps)
771
+ findings += [{**item, "file": f"{DOCS_DIR}/{item['file']}", "issue": "prose_stale"} for item in stale]
772
+ guidance = []
773
+ if len(stale) < len(findings):
774
+ guidance.append("Regenerate with project_manage action=docs generate=true: the skeleton follows the code and "
775
+ "the prose is kept; review the prose of the entries listed.")
776
+ if stale:
777
+ guidance.append("prose_stale: the code of the entry changed after its paragraph was written. Empty the block and "
778
+ "run docs write_prose=true, or rewrite it by hand and delete its code=\"...\" stamp, then "
779
+ "regenerate so the block records the code it now describes.")
780
+ return {"ok": not findings, "anchors": len(markers), "links": len(links), "findings_total": len(findings),
781
+ "findings": findings[:limit],
782
+ "guidance": " ".join(guidance) if findings else
783
+ "Every anchor matches the code, every link resolves and every entry paragraph was written for "
784
+ "the code of its function."}
785
+
786
+
787
+ def _read_pages(root, names):
788
+ """Text of the named generated documents that exist: relative file -> text."""
789
+ pages = {}
790
+ for rel in names:
791
+ path = os.path.join(root, DOCS_DIR, rel)
792
+ if os.path.isfile(path):
793
+ with open(path, encoding="utf-8") as stream:
794
+ pages[rel] = stream.read()
795
+ return pages
796
+
797
+
798
+ def anchors(root, symbol, view_id=None):
799
+ """Where a function is documented: its anchors in the generated documents with the drift state of each.
800
+
801
+ Parameters: root (str) project folder; symbol (str) name, Class.method or path::name; view_id (str or None).
802
+ Returns: dict with 'anchors' (file, line, id, level, current) and guidance when the documents do not exist.
803
+ """
804
+ if not isinstance(symbol, str) or not symbol.strip():
805
+ raise ValueError("Pass symbol: a function, Class.method or path::name.")
806
+ if not os.path.isdir(os.path.join(root, DOCS_DIR)):
807
+ return {"symbol": symbol, "anchors": [],
808
+ "guidance": f"No {DOCS_DIR} yet: generate it with project_manage action=docs generate=true."}
809
+ path, _sep, name = symbol.strip().rpartition("::")
810
+ path = path.replace("\\", "/").removeprefix("./")
811
+ markers, _anchors, _links = _scan(root)
812
+
813
+ def wanted(anchor):
814
+ where, _s, full = anchor.rpartition("::")
815
+ base = full.split("@")[0]
816
+ return (not path or where == path or where.endswith("/" + path)) and (base == name or base.split(".")[-1] == name)
817
+
818
+ found = [m for m in markers if wanted(m["id"])]
819
+ graph = code_graph.build_current(root, view_id, shared=True)
820
+ by_path = defaultdict(list)
821
+ for item in graph.get("symbols") or []:
822
+ by_path[item["path"]].append(item)
823
+ current, _stamps = _current_code(root, {m["id"].rpartition("::")[0] for m in found}, by_path)
824
+ return {"symbol": symbol.strip(),
825
+ "anchors": [{"file": f"{DOCS_DIR}/{m['file']}", "line": m["line"], "id": m["id"],
826
+ "level": "technical" if m["file"].startswith("technical/") else m["file"][:-3],
827
+ "current": current.get(m["id"]) == m["signature"]} for m in found]}
828
+
829
+
830
+ _MARKERS_CACHE = {}
831
+
832
+
833
+ def _markers_for(root, path):
834
+ """Signature markers of path's functions in the generated documents, cached by the documents' modification times."""
835
+ folder = os.path.join(root, DOCS_DIR)
836
+ if not os.path.isdir(folder):
837
+ return []
838
+ files = _generated_files(root)
839
+ stamp = tuple((rel, os.stat(os.path.join(folder, rel)).st_mtime_ns) for rel in files)
840
+ cached = _MARKERS_CACHE.get(root)
841
+ if not cached or cached[0] != stamp:
842
+ cached = (stamp, _scan(root)[0])
843
+ _MARKERS_CACHE[root] = cached
844
+ return [m for m in cached[1] if m["id"].rpartition("::")[0] == path]
845
+
846
+
847
+ def on_edit(root, rel, before, after_functions):
848
+ """Anchors of the edited file that this edit makes stale (matched the code before, not after).
849
+
850
+ Parameters: root (str) project folder; rel (str) project-relative path; before (str or None) text before the edit;
851
+ after_functions (list) doc_check functions of the file after the edit.
852
+ Returns: list of {'file', 'id', 'issue'}.
853
+ """
854
+ markers = _markers_for(root, rel)
855
+ if not markers:
856
+ return []
857
+ absolute = os.path.join(root, rel)
858
+ old = [f for f in doc_check.functions(absolute, before) if f.get("kind", "function") in ("function", "method")] \
859
+ if before else []
860
+ new = [f for f in after_functions if f.get("kind", "function") in ("function", "method")]
861
+ old_ids, new_ids = anchor_ids(old, rel), anchor_ids(new, rel)
862
+ then = {old_ids[f["start"]]: signature_hash(f["signature"]) for f in old}
863
+ now = {new_ids[f["start"]]: signature_hash(f["signature"]) for f in new}
864
+ stale = []
865
+ for marker in markers:
866
+ if then.get(marker["id"]) == marker["signature"] and now.get(marker["id"]) != marker["signature"]:
867
+ stale.append({"file": f"{DOCS_DIR}/{marker['file']}", "id": marker["id"],
868
+ "issue": "missing" if marker["id"] not in now else "signature_changed"})
869
+ return stale