@allansantos-dev/smart-tool 0.9.80 → 0.9.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -0
- package/NOTICE +1 -1
- package/README.md +59 -14
- package/agent_redirect.py +32 -0
- package/autostart.py +32 -25
- package/browser_bridge.py +320 -0
- package/browser_control.py +76 -39
- package/browser_service_client.py +1 -7
- package/client_hooks.py +42 -41
- package/code_graph.py +56 -15
- package/code_graph_java.mjs +21 -12
- package/code_health.py +19 -12
- package/code_impact.py +1 -1
- package/codex_config.py +3 -0
- package/config.py +35 -0
- package/daemon_launcher.py +37 -5
- package/doc_candidates.py +63 -0
- package/doc_check.py +9 -3
- package/doc_extras.py +2 -3
- package/doc_insights.py +26 -2
- package/doc_prose.py +59 -11
- package/doc_registry.py +241 -0
- package/doc_scope.py +205 -39
- package/doc_turn.py +205 -0
- package/doc_validate.py +52 -9
- package/document_read.py +4 -2
- package/dossier.py +40 -5
- package/edit_shape.py +119 -0
- package/endpoint_sync.py +3 -21
- package/engine_home.py +95 -0
- package/hook_call.py +132 -0
- package/hook_decision.py +337 -23
- package/hooks/pretool_router.py +36 -29
- package/hooks/turn_end.py +40 -0
- package/index_views.py +5 -0
- package/install.py +90 -36
- package/install_runtime.py +24 -10
- package/lazy_mcp_proxy.py +10 -9
- package/link_check.py +69 -0
- package/local_embedder.py +9 -2
- package/locales/pt-BR.json +72 -4
- package/model_breaker.py +6 -5
- package/model_client.py +1 -1
- package/model_health.py +24 -6
- package/notice_action.py +9 -0
- package/package.json +2 -2
- package/pr_comment.py +94 -0
- package/pr_impact.py +216 -35
- package/process_limits.py +17 -3
- package/project_admission.py +17 -15
- package/project_identity.py +19 -1
- package/project_scan.py +98 -0
- package/removed_callers.py +146 -0
- package/scout_access.py +48 -0
- package/setup_ui.py +33 -4
- package/shell_reads.py +67 -0
- package/smart_tool_daemon.py +149 -39
- package/tray.py +60 -9
- package/update_splash.py +2 -2
- package/venv_launch.py +67 -0
- package/version.py +1 -1
- package/web_adapters/node/engine-home.mjs +42 -0
- package/web_adapters/node/package-lock.json +13 -0
- package/web_adapters/node/package.json +1 -0
- package/web_document_graph.py +12 -1
package/doc_scope.py
CHANGED
|
@@ -29,6 +29,7 @@ import code_graph
|
|
|
29
29
|
import config
|
|
30
30
|
import doc_check
|
|
31
31
|
import doc_cli
|
|
32
|
+
import doc_registry
|
|
32
33
|
import index_profile
|
|
33
34
|
import index_scope
|
|
34
35
|
import layers
|
|
@@ -46,7 +47,10 @@ MIN_ROOT_FLOW = 3
|
|
|
46
47
|
MAX_ROOTS = 40
|
|
47
48
|
PROSE = re.compile(r'<!-- smart-tool:prose (.+?)(?: code="[0-9a-f]{12}")? -->\n(.*?)<!-- /smart-tool:prose -->', re.S)
|
|
48
49
|
STAMPED_PROSE = re.compile(r'<!-- smart-tool:prose (.+?)(?: code="([0-9a-f]{12})")? -->\n(.*?)<!-- /smart-tool:prose -->', re.S)
|
|
49
|
-
MARKER = re.compile(r'<!-- smart-tool:symbol id="([^"]+)" signature="([0-9a-f]{12})"
|
|
50
|
+
MARKER = re.compile(r'<!-- smart-tool:symbol id="([^"]+)" signature="([0-9a-f]{12})" body="([0-9a-f]{12})" '
|
|
51
|
+
r'docstring="([0-9a-f]{12})" -->')
|
|
52
|
+
ANY_MARKER = re.compile(r"<!-- smart-tool:symbol .*?-->")
|
|
53
|
+
STAMPS = ("signature", "body", "docstring")
|
|
50
54
|
LINK = re.compile(r"\]\(<([^>#]*)#([^>]+)>\)")
|
|
51
55
|
ANCHOR = re.compile(r'<a id="([^"]+)"></a>')
|
|
52
56
|
_SPACE = re.compile(r"\s+")
|
|
@@ -72,17 +76,65 @@ def code_stamp(source, function):
|
|
|
72
76
|
return signature_hash("\n".join(source.split("\n")[function["start"] - 1:function["end"]]))
|
|
73
77
|
|
|
74
78
|
|
|
79
|
+
def function_stamps(source, function):
|
|
80
|
+
"""Signature, body and docstring stamps of a function: what a technical entry records it was written from.
|
|
81
|
+
|
|
82
|
+
Parameters: source (str) text of the function's file; function (dict) doc_check function with 'start', 'end',
|
|
83
|
+
'signature', 'docstring' and, for Python, 'doc_lines' (the docstring's lines, left out of the body stamp).
|
|
84
|
+
Returns: dict signature/body/docstring -> str of 12 hex digits.
|
|
85
|
+
"""
|
|
86
|
+
first, last = function.get("doc_lines") or (0, -1)
|
|
87
|
+
body = [line for number, line in enumerate(source.split("\n")[function["start"] - 1:function["end"]], function["start"])
|
|
88
|
+
if not first <= number <= last]
|
|
89
|
+
return {"signature": signature_hash(function["signature"]), "body": signature_hash("\n".join(body)),
|
|
90
|
+
"docstring": signature_hash(function.get("docstring") or "")}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def symbol_marker(anchor, stamps):
|
|
94
|
+
"""The symbol marker of an anchor with the stamps of the code it was generated from."""
|
|
95
|
+
return (f'<!-- smart-tool:symbol id="{_escape(anchor)}" signature="{stamps["signature"]}" body="{stamps["body"]}" '
|
|
96
|
+
f'docstring="{stamps["docstring"]}" -->')
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def changed_parts(recorded, now):
|
|
100
|
+
"""The stamps (signature, body, docstring) that differ between a marker and the code now, in that order."""
|
|
101
|
+
return [part for part in STAMPS if recorded.get(part) != now.get(part)]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def drift_issue(parts):
|
|
105
|
+
"""The issue name of a marker whose stamps differ: signature_changed before body_changed before docstring_changed."""
|
|
106
|
+
return f"{parts[0]}_changed"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def flow_stamp(model, anchor):
|
|
110
|
+
"""Stamp of a feature's prose: the code of the entry point and of every function its flow reaches, so a paragraph
|
|
111
|
+
goes stale when a function deep in the flow changes, not only when the entry point's own body does.
|
|
112
|
+
|
|
113
|
+
Parameters: model (_Model); anchor (str) entry point. Returns: str of 12 hex digits.
|
|
114
|
+
"""
|
|
115
|
+
stamps = []
|
|
116
|
+
for reached in [anchor] + sorted(model.flow(anchor)):
|
|
117
|
+
function = model.function_of[reached]
|
|
118
|
+
stamps.append(f"{reached}={code_stamp(model.read(function['path']) or '', function)}")
|
|
119
|
+
return signature_hash("\n".join(stamps))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def prose_stamps(model):
|
|
123
|
+
"""Flow stamp of every entry point: the `current` argument of stale_prose."""
|
|
124
|
+
return {anchor: flow_stamp(model, anchor) for anchor in model.entry_points()}
|
|
125
|
+
|
|
126
|
+
|
|
75
127
|
def prose_open(key, stamp=None):
|
|
76
128
|
"""Opening marker of the prose block `key`, with the code stamp it was written for when there is one."""
|
|
77
129
|
return f"<!-- smart-tool:prose {key}" + (f' code="{stamp}"' if stamp else "") + " -->"
|
|
78
130
|
|
|
79
131
|
|
|
80
132
|
def stale_prose(pages, current):
|
|
81
|
-
"""Written entry-point prose of functional.md and simple.md whose
|
|
82
|
-
a block without a stamp included: the paragraph may describe behavior the
|
|
133
|
+
"""Written entry-point prose of functional.md and simple.md whose stamp is not the flow stamp of its entry point
|
|
134
|
+
now, a block without a stamp included: the paragraph may describe behavior the feature no longer has.
|
|
83
135
|
|
|
84
|
-
Parameters: pages (dict) relative file -> text; current (dict) anchor ->
|
|
85
|
-
gone is left out, its signature marker already reports it).
|
|
136
|
+
Parameters: pages (dict) relative file -> text; current (dict) anchor -> flow_stamp of the entry point now (an
|
|
137
|
+
entry point gone is left out, its signature marker already reports it).
|
|
86
138
|
Returns: list of dicts with file, line and id.
|
|
87
139
|
"""
|
|
88
140
|
found = []
|
|
@@ -180,6 +232,7 @@ class _Model:
|
|
|
180
232
|
self._command_lines = None
|
|
181
233
|
self._flows = {}
|
|
182
234
|
self._runners = {}
|
|
235
|
+
self._possible = None
|
|
183
236
|
self.roots_hidden = 0
|
|
184
237
|
self.graph = code_graph.build_current(root, view_id, shared=True)
|
|
185
238
|
self.scope = index_scope.load_scope(root) or {}
|
|
@@ -209,18 +262,40 @@ class _Model:
|
|
|
209
262
|
self.function_of[anchor] = {**function, "path": path, "id": anchor}
|
|
210
263
|
self.symbol_of[anchor] = symbol
|
|
211
264
|
self.callers, self.callees = defaultdict(set), defaultdict(set)
|
|
212
|
-
self.used_by_module = set()
|
|
265
|
+
self.used_by_module, self.declared = set(), set()
|
|
213
266
|
for call in self.graph.get("calls") or []:
|
|
214
267
|
source, target = self.anchor_of.get(call["source"]), self.anchor_of.get(call["target"])
|
|
215
268
|
if source and target and source != target:
|
|
216
269
|
self.callers[target].add(source)
|
|
217
270
|
self.callees[source].add(target)
|
|
271
|
+
if call.get("resolution") == "declared":
|
|
272
|
+
self.declared.add((source, target))
|
|
218
273
|
elif target and not source and self.kind.get(call["source"].rpartition("::")[0]) == "code":
|
|
219
274
|
self.used_by_module.add(target)
|
|
220
275
|
annotated = bool(self.rule) and any(layer["annotations"] for layer in self.rule.layers)
|
|
221
276
|
self.layer_of = {path: self.rule.assign(path, self.read(path) if annotated else None) if self.rule else None
|
|
222
277
|
for path in by_path}
|
|
223
278
|
|
|
279
|
+
def possible(self):
|
|
280
|
+
"""Embedder candidates recorded in the registry that still apply (a public function with no path from any entry
|
|
281
|
+
point, a feature that still exists): (by function {anchor: [(entry, score)]}, by feature {entry: [(anchor,
|
|
282
|
+
score)]}). Read once; never a confirmed link."""
|
|
283
|
+
if self._possible is None:
|
|
284
|
+
entries, reached = set(self.entry_points()), features(self)
|
|
285
|
+
by_function, by_feature = defaultdict(list), defaultdict(list)
|
|
286
|
+
for row in sorted(doc_registry.candidates(self.root), key=lambda r: (-r["score"], r["feature_id"], r["fn_id"])):
|
|
287
|
+
anchor, entry = row["fn_id"], row["feature_id"]
|
|
288
|
+
if anchor in self.function_of and anchor not in reached and entry in entries:
|
|
289
|
+
by_function[anchor].append((entry, row["score"]))
|
|
290
|
+
by_feature[entry].append((anchor, row["score"]))
|
|
291
|
+
self._possible = (dict(by_function), dict(by_feature))
|
|
292
|
+
return self._possible
|
|
293
|
+
|
|
294
|
+
def stamps(self, anchor):
|
|
295
|
+
"""Signature, body and docstring stamps of a function as it is now (see function_stamps)."""
|
|
296
|
+
function = self.function_of[anchor]
|
|
297
|
+
return function_stamps(self.read(function["path"]), function)
|
|
298
|
+
|
|
224
299
|
def read(self, path):
|
|
225
300
|
"""Text of a project file, read once; None when it cannot be read as UTF-8."""
|
|
226
301
|
if path not in self._texts:
|
|
@@ -389,14 +464,60 @@ def _generated_files(root):
|
|
|
389
464
|
return sorted(found)
|
|
390
465
|
|
|
391
466
|
|
|
392
|
-
def
|
|
467
|
+
def features(model):
|
|
468
|
+
"""Features each function belongs to: the entry points whose flow reaches it, and the entry point itself.
|
|
469
|
+
|
|
470
|
+
Returns: dict anchor -> list of (entry anchor, hops, origin), origin 'declared' when a call on the path goes through
|
|
471
|
+
an interface the project declares, else 'graph'.
|
|
472
|
+
"""
|
|
473
|
+
found = defaultdict(list)
|
|
474
|
+
for entry in model.entry_points():
|
|
475
|
+
hops, origin = {entry: 0}, {entry: "graph"}
|
|
476
|
+
found[entry].append((entry, 0, "graph"))
|
|
477
|
+
for caller, callee in model.flow_edges(entry):
|
|
478
|
+
hops[callee] = hops[caller] + 1
|
|
479
|
+
origin[callee] = "declared" if (caller, callee) in model.declared or origin[caller] == "declared" else "graph"
|
|
480
|
+
found[callee].append((entry, hops[callee], origin[callee]))
|
|
481
|
+
return found
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def feature_links(model):
|
|
485
|
+
"""Rows of the feature link table: one per (feature, function) pair the graph confirms."""
|
|
486
|
+
return [{"feature_id": entry, "fn_id": anchor, "layer": model.layer_of.get(model.function_of[anchor]["path"]),
|
|
487
|
+
"hops": hops, "origin": origin, "score": None, "status": "confirmed"}
|
|
488
|
+
for anchor, items in sorted(features(model).items()) for entry, hops, origin in items]
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _feature_line(model, anchor, belongs, entries, page):
|
|
492
|
+
"""The technical entry's '- Feature:' line: a link per feature its function belongs to, or 'none'."""
|
|
493
|
+
if not belongs:
|
|
494
|
+
return "- Feature: none"
|
|
495
|
+
target = posixpath.relpath("functional.md", posixpath.dirname(page) or ".")
|
|
496
|
+
return "- Feature: " + ", ".join(entry_link(model, entry, entries, target) for entry, _h, _o in belongs)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _possible_score(score):
|
|
500
|
+
return f"(embedder score {score:.2f}, not confirmed)"
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _possible_feature_line(model, anchor, entries, page):
|
|
504
|
+
"""The technical entry's '- Possible feature:' line from the embedder candidates, or None without any."""
|
|
505
|
+
found = model.possible()[0].get(anchor)
|
|
506
|
+
if not found:
|
|
507
|
+
return None
|
|
508
|
+
target = posixpath.relpath("functional.md", posixpath.dirname(page) or ".")
|
|
509
|
+
return "- Possible feature: " + ", ".join(f"{entry_link(model, entry, entries, target)} {_possible_score(score)}"
|
|
510
|
+
for entry, score in found)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _links(model, anchors, page_of, from_page, limit=MAX_LINKS):
|
|
393
514
|
shown = []
|
|
394
|
-
for anchor in sorted(anchors)[:
|
|
515
|
+
for anchor in sorted(anchors)[:limit]:
|
|
395
516
|
name = model.function_of[anchor]["name"]
|
|
396
517
|
target = page_of.get(anchor)
|
|
397
518
|
shown.append(_link(name, posixpath.relpath(target, posixpath.dirname(from_page) or "."), anchor) if target
|
|
398
519
|
else f"`{name}` ({model.function_of[anchor]['path']})")
|
|
399
|
-
more = len(anchors) -
|
|
520
|
+
more = len(anchors) - limit
|
|
400
521
|
return ", ".join(shown) + (f" (+{more} more)" if more > 0 else "")
|
|
401
522
|
|
|
402
523
|
|
|
@@ -406,6 +527,7 @@ def _technical(model, kept):
|
|
|
406
527
|
page_of, titles = _pages(model, public)
|
|
407
528
|
truncated = TRUNCATED if model.graph.get("truncated") else ""
|
|
408
529
|
pages, by_page = {}, defaultdict(list)
|
|
530
|
+
belongs, entries = features(model), model.entry_points()
|
|
409
531
|
for anchor in public:
|
|
410
532
|
by_page[page_of[anchor]].append(anchor)
|
|
411
533
|
for page, anchors in sorted(by_page.items()):
|
|
@@ -425,9 +547,13 @@ def _technical(model, kept):
|
|
|
425
547
|
function = model.function_of[anchor]
|
|
426
548
|
tests = model.tests_of(anchor)
|
|
427
549
|
lines += [f'<a id="{_escape(anchor)}"></a>', f"### `{function['name']}`", "",
|
|
428
|
-
|
|
550
|
+
symbol_marker(anchor, model.stamps(anchor)),
|
|
429
551
|
"", f"`{function['signature'] or function['name']}` (line {function['start']})", "",
|
|
430
|
-
function["doc"] or "*No docstring yet.*", ""
|
|
552
|
+
function["doc"] or "*No docstring yet.*", "",
|
|
553
|
+
_feature_line(model, anchor, belongs.get(anchor), entries, page)]
|
|
554
|
+
possible = _possible_feature_line(model, anchor, entries, page)
|
|
555
|
+
if possible:
|
|
556
|
+
lines.append(possible)
|
|
431
557
|
callers = {c for c in model.callers[anchor] if model.kind[model.function_of[c]["path"]] == "code"}
|
|
432
558
|
if callers:
|
|
433
559
|
lines.append(f"- Called by: {_links(model, callers, page_of, page)}")
|
|
@@ -589,16 +715,20 @@ def _functional(model, kept, page_of):
|
|
|
589
715
|
for anchor in entries:
|
|
590
716
|
function = model.function_of[anchor]
|
|
591
717
|
lines += [f'<a id="{_escape(anchor)}"></a>', f"## {entry_title(model, anchor, entries)}", "",
|
|
592
|
-
|
|
718
|
+
symbol_marker(anchor, model.stamps(anchor)),
|
|
593
719
|
"", f"Entry point in `{function['path']}` (line {function['start']})."
|
|
594
720
|
+ (f" Technical entry: {_link(function['name'], page_of[anchor], anchor)}." if anchor in page_of else ""),
|
|
595
|
-
"", _prose(anchor, own,
|
|
721
|
+
"", _prose(anchor, own, flow_stamp(model, anchor)).rstrip("\n"), ""]
|
|
596
722
|
groups = defaultdict(list)
|
|
597
723
|
for reached in model.flow(anchor):
|
|
598
724
|
groups[model.layer_of.get(model.function_of[reached]["path"]) or "other"].append(reached)
|
|
599
725
|
for layer in order + ["other"]:
|
|
600
726
|
if groups.get(layer):
|
|
601
|
-
lines.append(f"- {layer}: {_links(model, groups[layer], page_of, 'functional.md')}")
|
|
727
|
+
lines.append(f"- {layer}: {_links(model, groups[layer], page_of, 'functional.md', MAX_FLOW)}")
|
|
728
|
+
possible = [(a, s) for a, s in model.possible()[1].get(anchor, ()) if a in page_of]
|
|
729
|
+
if possible:
|
|
730
|
+
lines.append("- possible: " + ", ".join(f"{_link(model.function_of[a]['name'], page_of[a], a)} "
|
|
731
|
+
f"{_possible_score(s)}" for a, s in possible))
|
|
602
732
|
lines.append("")
|
|
603
733
|
lines += _command_lines(model, entries)
|
|
604
734
|
return "\n".join(lines).rstrip("\n") + "\n", entries
|
|
@@ -614,7 +744,7 @@ def _simple(model, kept, entries, vocabulary=()):
|
|
|
614
744
|
for anchor in entries:
|
|
615
745
|
function = model.function_of[anchor]
|
|
616
746
|
lines += [f"### {entry_title(model, anchor, entries)}", "",
|
|
617
|
-
_prose(anchor, own,
|
|
747
|
+
_prose(anchor, own, flow_stamp(model, anchor)).rstrip("\n"),
|
|
618
748
|
f"[Details](<functional.md#{_escape(anchor)}>)", ""]
|
|
619
749
|
return "\n".join(lines).rstrip("\n") + "\n"
|
|
620
750
|
|
|
@@ -670,6 +800,8 @@ def generate(root, view_id=None, limit=30, model=None):
|
|
|
670
800
|
raise ValueError(f"{full} exists and was not written by Smart Tool; move it before generating.")
|
|
671
801
|
for rel, text in files.items():
|
|
672
802
|
full = os.path.join(folder, rel)
|
|
803
|
+
if rel in owned and _same_text(full, text):
|
|
804
|
+
continue
|
|
673
805
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
674
806
|
atomic_io.write_text(full, text)
|
|
675
807
|
import doc_validate
|
|
@@ -702,6 +834,12 @@ def generate(root, view_id=None, limit=30, model=None):
|
|
|
702
834
|
"every claim again, the prose included."}
|
|
703
835
|
|
|
704
836
|
|
|
837
|
+
def _same_text(path, text):
|
|
838
|
+
"""True when the file at path already holds text, so a regeneration leaves it untouched."""
|
|
839
|
+
with open(path, encoding="utf-8", newline="") as stream:
|
|
840
|
+
return stream.read() == text
|
|
841
|
+
|
|
842
|
+
|
|
705
843
|
def _scan(root):
|
|
706
844
|
"""Anchors, signature markers and links of the generated documents: (markers, anchors per file, links)."""
|
|
707
845
|
markers, anchors, links = [], defaultdict(set), []
|
|
@@ -709,17 +847,21 @@ def _scan(root):
|
|
|
709
847
|
for rel in _generated_files(root):
|
|
710
848
|
with open(os.path.join(folder, rel), encoding="utf-8") as stream:
|
|
711
849
|
for number, line in enumerate(stream, 1):
|
|
712
|
-
|
|
713
|
-
|
|
850
|
+
found = MARKER.findall(line)
|
|
851
|
+
if len(found) != len(ANY_MARKER.findall(line)):
|
|
852
|
+
raise ValueError(f"{DOCS_DIR}/{rel}:{number} has a symbol marker in an older format (without the body "
|
|
853
|
+
"and docstring stamps); regenerate the documents with project_manage action=docs "
|
|
854
|
+
"generate=true, which keeps the written prose.")
|
|
855
|
+
for anchor, *digests in found:
|
|
856
|
+
markers.append({"file": rel, "line": number, "id": html.unescape(anchor), **dict(zip(STAMPS, digests))})
|
|
714
857
|
anchors[rel].update(html.unescape(a) for a in ANCHOR.findall(line))
|
|
715
858
|
links += [{"file": rel, "line": number, "target": target, "anchor": html.unescape(anchor)}
|
|
716
859
|
for target, anchor in LINK.findall(line)]
|
|
717
860
|
return markers, anchors, links
|
|
718
861
|
|
|
719
862
|
|
|
720
|
-
def
|
|
721
|
-
"""
|
|
722
|
-
current, stamps = {}, {}
|
|
863
|
+
def disk_functions(root, paths, symbols_by_path):
|
|
864
|
+
"""Yields (path, source, function, anchor_id) for each function or method of paths as it is on disk now."""
|
|
723
865
|
for path in paths:
|
|
724
866
|
try:
|
|
725
867
|
with open(os.path.join(root, path), encoding="utf-8") as stream:
|
|
@@ -729,33 +871,41 @@ def _current_code(root, paths, symbols_by_path):
|
|
|
729
871
|
found = [f for f in doc_check.functions(path, source, symbols_by_path.get(path))
|
|
730
872
|
if f.get("kind", "function") in ("function", "method")]
|
|
731
873
|
ids = anchor_ids(found, path)
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
874
|
+
for function in found:
|
|
875
|
+
yield path, source, function, ids[function["start"]]
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def _current_code(root, paths, symbols_by_path):
|
|
879
|
+
"""Stamps (function_stamps) of the functions in paths as they are on disk now, by anchor id."""
|
|
880
|
+
return {fn_id: function_stamps(source, function)
|
|
881
|
+
for _path, source, function, fn_id in disk_functions(root, paths, symbols_by_path)}
|
|
735
882
|
|
|
736
883
|
|
|
737
884
|
def check(root, view_id=None, limit=30):
|
|
738
|
-
"""Drift of the generated documents: anchors whose function is gone or changed its signature
|
|
739
|
-
links whose target anchor does not exist.
|
|
885
|
+
"""Drift of the generated documents: anchors whose function is gone or changed its signature, body or docstring
|
|
886
|
+
since generation, and links whose target anchor does not exist.
|
|
740
887
|
|
|
741
888
|
Parameters: root (str) project folder; view_id (str or None) index view; limit (int) findings listed.
|
|
742
|
-
Returns: dict with 'ok', 'findings' (file, line, id, issue: missing | signature_changed |
|
|
889
|
+
Returns: dict with 'ok', 'findings' (file, line, id, issue: missing | signature_changed | body_changed |
|
|
890
|
+
docstring_changed | broken_link | prose_stale, and 'changed' for a drifted anchor) and counts.
|
|
743
891
|
"""
|
|
744
892
|
if not os.path.isdir(os.path.join(root, DOCS_DIR)):
|
|
745
893
|
raise ValueError(f"No {DOCS_DIR} in the project yet: generate it with project_manage action=docs generate=true.")
|
|
746
894
|
markers, anchors, links = _scan(root)
|
|
747
|
-
|
|
895
|
+
model = _Model(root, view_id)
|
|
748
896
|
by_path = defaultdict(list)
|
|
749
|
-
for symbol in graph.get("symbols") or []:
|
|
897
|
+
for symbol in model.graph.get("symbols") or []:
|
|
750
898
|
by_path[symbol["path"]].append(symbol)
|
|
751
899
|
paths = {m["id"].rpartition("::")[0] for m in markers}
|
|
752
|
-
current
|
|
900
|
+
current = _current_code(root, paths, by_path)
|
|
753
901
|
findings = []
|
|
754
902
|
for marker in markers:
|
|
755
903
|
now = current.get(marker["id"])
|
|
756
|
-
if now
|
|
904
|
+
parts = changed_parts(marker, now) if now else []
|
|
905
|
+
if now is None or parts:
|
|
757
906
|
findings.append({"file": f"{DOCS_DIR}/{marker['file']}", "line": marker["line"], "id": marker["id"],
|
|
758
|
-
"issue": "missing" if now is None else
|
|
907
|
+
"issue": "missing" if now is None else drift_issue(parts),
|
|
908
|
+
**({"changed": parts} if parts else {})})
|
|
759
909
|
for link in links:
|
|
760
910
|
target = posixpath.normpath(posixpath.join(posixpath.dirname(link["file"]), link["target"])) \
|
|
761
911
|
if link["target"] else link["file"]
|
|
@@ -767,14 +917,15 @@ def check(root, view_id=None, limit=30):
|
|
|
767
917
|
links.append(link)
|
|
768
918
|
if link["anchor"] not in anchors.get(link["target"], ()):
|
|
769
919
|
findings.append({"file": link["file"], "line": link["line"], "id": link["anchor"], "issue": "broken_link"})
|
|
770
|
-
stale = stale_prose(_read_pages(root, ("functional.md", "simple.md")),
|
|
920
|
+
stale = stale_prose(_read_pages(root, ("functional.md", "simple.md")), prose_stamps(model))
|
|
771
921
|
findings += [{**item, "file": f"{DOCS_DIR}/{item['file']}", "issue": "prose_stale"} for item in stale]
|
|
772
922
|
guidance = []
|
|
773
923
|
if len(stale) < len(findings):
|
|
774
924
|
guidance.append("Regenerate with project_manage action=docs generate=true: the skeleton follows the code and "
|
|
775
925
|
"the prose is kept; review the prose of the entries listed.")
|
|
776
926
|
if stale:
|
|
777
|
-
guidance.append("prose_stale: the code of the
|
|
927
|
+
guidance.append("prose_stale: the code of the feature (its entry point or a function its flow reaches) changed "
|
|
928
|
+
"after its paragraph was written. Empty the block and "
|
|
778
929
|
"run docs write_prose=true, or rewrite it by hand and delete its code=\"...\" stamp, then "
|
|
779
930
|
"regenerate so the block records the code it now describes.")
|
|
780
931
|
return {"ok": not findings, "anchors": len(markers), "links": len(links), "findings_total": len(findings),
|
|
@@ -820,11 +971,11 @@ def anchors(root, symbol, view_id=None):
|
|
|
820
971
|
by_path = defaultdict(list)
|
|
821
972
|
for item in graph.get("symbols") or []:
|
|
822
973
|
by_path[item["path"]].append(item)
|
|
823
|
-
current
|
|
974
|
+
current = _current_code(root, {m["id"].rpartition("::")[0] for m in found}, by_path)
|
|
824
975
|
return {"symbol": symbol.strip(),
|
|
825
976
|
"anchors": [{"file": f"{DOCS_DIR}/{m['file']}", "line": m["line"], "id": m["id"],
|
|
826
977
|
"level": "technical" if m["file"].startswith("technical/") else m["file"][:-3],
|
|
827
|
-
"current": current.get(m["id"])
|
|
978
|
+
"current": not changed_parts(m, current.get(m["id"]) or {})} for m in found]}
|
|
828
979
|
|
|
829
980
|
|
|
830
981
|
_MARKERS_CACHE = {}
|
|
@@ -859,11 +1010,26 @@ def on_edit(root, rel, before, after_functions):
|
|
|
859
1010
|
if before else []
|
|
860
1011
|
new = [f for f in after_functions if f.get("kind", "function") in ("function", "method")]
|
|
861
1012
|
old_ids, new_ids = anchor_ids(old, rel), anchor_ids(new, rel)
|
|
862
|
-
|
|
863
|
-
|
|
1013
|
+
source_before, source_after = before or "", _source_of(after_functions, absolute)
|
|
1014
|
+
then = {old_ids[f["start"]]: function_stamps(source_before, f) for f in old}
|
|
1015
|
+
now = {new_ids[f["start"]]: function_stamps(source_after, f) for f in new} if source_after is not None else {}
|
|
864
1016
|
stale = []
|
|
865
1017
|
for marker in markers:
|
|
866
|
-
if
|
|
1018
|
+
if marker["id"] not in then or changed_parts(marker, then[marker["id"]]):
|
|
1019
|
+
continue
|
|
1020
|
+
parts = changed_parts(marker, now[marker["id"]]) if marker["id"] in now else []
|
|
1021
|
+
if marker["id"] not in now or parts:
|
|
867
1022
|
stale.append({"file": f"{DOCS_DIR}/{marker['file']}", "id": marker["id"],
|
|
868
|
-
"issue": "missing" if marker["id"] not in now else
|
|
1023
|
+
"issue": "missing" if marker["id"] not in now else drift_issue(parts),
|
|
1024
|
+
**({"changed": parts} if parts else {})})
|
|
869
1025
|
return stale
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def _source_of(functions, absolute):
|
|
1029
|
+
"""Text of the edited file after the edit (the edit hook runs once it is on disk); None when it cannot be read and
|
|
1030
|
+
the file still has functions, so their stamps cannot be compared."""
|
|
1031
|
+
try:
|
|
1032
|
+
with open(absolute, encoding="utf-8") as stream:
|
|
1033
|
+
return stream.read()
|
|
1034
|
+
except (OSError, UnicodeDecodeError):
|
|
1035
|
+
return "" if not functions else None
|
package/doc_turn.py
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"""End-of-turn documentation step, run by the Claude Code Stop hook (hooks/turn_end.py, a `command` hook) through the
|
|
2
|
+
daemon route /hooks/stop, outside the search executor.
|
|
3
|
+
|
|
4
|
+
Flow: a continuation (stop_hook_active) or a folder without a generated docs/scope exits at once; otherwise the files
|
|
5
|
+
the turn touched (noted by the PreToolUse of Edit, MultiEdit and Write) and the documented files whose size or mtime
|
|
6
|
+
changed (subagents and shell writes) are re-stamped against the page markers in the registry (doc_registry), at most
|
|
7
|
+
MAX_FUNCTIONS pending functions and MAX_NODES chain nodes are claimed, the feature links are recorded (the graph's,
|
|
8
|
+
plus embedder candidates for public functions no entry point reaches, never confirmed by the embedder), the pages are
|
|
9
|
+
regenerated without a model (only pages whose text changes are written) and the registry is settled. Walking the
|
|
10
|
+
chain back, the written functional and simple paragraphs of the features those functions belong to whose flow stamp
|
|
11
|
+
changed are rewritten by the docs_model, within the same node budget. When a public function's body changed and its
|
|
12
|
+
docstring did not, the hook blocks once with the short list the session agent must review: the Smart Tool never
|
|
13
|
+
writes the user's code. What could not be done (no Docs model, a rejected paragraph, the embeddings home down) reaches
|
|
14
|
+
the user as a systemMessage, never silently.
|
|
15
|
+
"""
|
|
16
|
+
import os
|
|
17
|
+
import threading
|
|
18
|
+
from collections import defaultdict
|
|
19
|
+
|
|
20
|
+
import doc_candidates
|
|
21
|
+
import doc_check
|
|
22
|
+
import doc_prose
|
|
23
|
+
import doc_registry
|
|
24
|
+
import doc_scope
|
|
25
|
+
|
|
26
|
+
MAX_FUNCTIONS = 20
|
|
27
|
+
MAX_NODES = 60
|
|
28
|
+
PROSE_BUDGET_S = 40
|
|
29
|
+
ORIGIN = "[hook-origin=smart-tool]"
|
|
30
|
+
_TOUCHED = defaultdict(set)
|
|
31
|
+
_SEEN_SESSIONS = set()
|
|
32
|
+
_LOCK = threading.Lock()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def project_root(path):
|
|
36
|
+
"""Nearest folder from path upward whose docs/scope index was written by the generator; None when there is none."""
|
|
37
|
+
current = os.path.abspath(path)
|
|
38
|
+
while True:
|
|
39
|
+
index = os.path.join(current, doc_scope.DOCS_DIR, "README.md")
|
|
40
|
+
if os.path.isfile(index):
|
|
41
|
+
with open(index, encoding="utf-8") as stream:
|
|
42
|
+
if stream.readline().rstrip("\n") == doc_scope.GENERATED:
|
|
43
|
+
return current
|
|
44
|
+
parent = os.path.dirname(current)
|
|
45
|
+
if parent == current:
|
|
46
|
+
return None
|
|
47
|
+
current = parent
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def note_tool(payload):
|
|
51
|
+
"""PreToolUse: note the file an Edit, MultiEdit or Write touches for the session's next Stop, and on the session's
|
|
52
|
+
first call resume the documentation work a stopped run left behind (doc_registry.recover)."""
|
|
53
|
+
session, cwd = payload.get("session_id") or "", payload.get("cwd") or ""
|
|
54
|
+
tool, given = payload.get("tool_name"), (payload.get("tool_input") or {}).get("file_path")
|
|
55
|
+
with _LOCK:
|
|
56
|
+
first = bool(session) and session not in _SEEN_SESSIONS
|
|
57
|
+
_SEEN_SESSIONS.add(session)
|
|
58
|
+
if tool in ("Edit", "MultiEdit", "Write") and given:
|
|
59
|
+
path = given if os.path.isabs(given) else os.path.join(cwd, given)
|
|
60
|
+
root = project_root(os.path.dirname(path))
|
|
61
|
+
if root:
|
|
62
|
+
with _LOCK:
|
|
63
|
+
_TOUCHED[session].add((root, os.path.relpath(path, root).replace(os.sep, "/")))
|
|
64
|
+
root = project_root(cwd) if first and cwd else None
|
|
65
|
+
if root:
|
|
66
|
+
resume(root)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def resume(root):
|
|
70
|
+
"""Return the expired in_progress rows of a project to pending. Returns: int rows recovered."""
|
|
71
|
+
conn = doc_registry.connect()
|
|
72
|
+
try:
|
|
73
|
+
return doc_registry.recover(conn, root)
|
|
74
|
+
finally:
|
|
75
|
+
conn.close()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def resume_all():
|
|
79
|
+
"""Daemon start: resume every tracked project that still has its generated documents. Returns: int rows recovered."""
|
|
80
|
+
conn = doc_registry.connect()
|
|
81
|
+
try:
|
|
82
|
+
return sum(doc_registry.recover(conn, root) for root in doc_registry.projects(conn)
|
|
83
|
+
if os.path.isdir(os.path.join(root, doc_scope.DOCS_DIR)))
|
|
84
|
+
finally:
|
|
85
|
+
conn.close()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def on_stop(payload, view_id=None):
|
|
89
|
+
"""Stop hook answer: {} when there is nothing to do; {decision: block, reason} once per turn with the functions whose
|
|
90
|
+
docstring the session agent must check against their new body; a systemMessage with what the step could not do."""
|
|
91
|
+
if payload.get("stop_hook_active"):
|
|
92
|
+
return {}
|
|
93
|
+
cwd = payload.get("cwd") or ""
|
|
94
|
+
root = project_root(cwd) if cwd else None
|
|
95
|
+
if not root:
|
|
96
|
+
return {}
|
|
97
|
+
with _LOCK:
|
|
98
|
+
touched = {rel for owner, rel in _TOUCHED.pop(payload.get("session_id") or "", set()) if owner == root}
|
|
99
|
+
result = run(root, touched, view_id)
|
|
100
|
+
answer = {}
|
|
101
|
+
if result["review"]:
|
|
102
|
+
lines = [f"- `{item['fn_id']}`: the body changed and the docstring did not"
|
|
103
|
+
+ (f"; features: {', '.join(item['features'])}" if item["features"] else "") for item in result["review"]]
|
|
104
|
+
answer = {"decision": "block",
|
|
105
|
+
"reason": "Smart Tool documentation: check each docstring below against the function's new body and fix "
|
|
106
|
+
"it in the code if it no longer describes it (the technical pages were already updated):\n"
|
|
107
|
+
+ "\n".join(lines)}
|
|
108
|
+
if result["notes"]:
|
|
109
|
+
answer["systemMessage"] = f"{ORIGIN} documentation step: " + " ".join(result["notes"])
|
|
110
|
+
return answer
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _functions_now(root, paths, by_path, public):
|
|
114
|
+
"""Stamps of the functions of paths on disk: fn_id -> {path, public, has_docstring, signature, body, docstring}."""
|
|
115
|
+
return {fn_id: {"path": path, "public": fn_id in public, "has_docstring": function["documented"],
|
|
116
|
+
**doc_scope.function_stamps(source, function)}
|
|
117
|
+
for path, source, function, fn_id in doc_scope.disk_functions(
|
|
118
|
+
root, [p for p in paths if doc_check.supported(p)], by_path)}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def run(root, touched=(), view_id=None):
|
|
122
|
+
"""One end-of-turn pass over a project (see the module docstring).
|
|
123
|
+
|
|
124
|
+
Parameters: root (str) project with a generated docs/scope; touched (iterable of project-relative paths).
|
|
125
|
+
Returns: dict with claimed (fn_id list), settled, still_pending, review (fn_id, features) lists, candidates (new
|
|
126
|
+
candidate rows), prose (doc_prose.rewrite_stale result or None) and notes (what could not be done, for the user).
|
|
127
|
+
"""
|
|
128
|
+
conn = doc_registry.connect()
|
|
129
|
+
try:
|
|
130
|
+
doc_registry.recover(conn, root)
|
|
131
|
+
technical = {m["id"]: m for m in doc_scope._scan(root)[0] if m["file"].startswith("technical/")}
|
|
132
|
+
changed = doc_registry.changed_files(conn, root, {f.rpartition("::")[0] for f in technical} | set(touched))
|
|
133
|
+
model = doc_scope._Model(root, view_id)
|
|
134
|
+
by_path = defaultdict(list)
|
|
135
|
+
for symbol in model.graph.get("symbols") or []:
|
|
136
|
+
by_path[symbol["path"]].append(symbol)
|
|
137
|
+
public = set(model.public())
|
|
138
|
+
now = _functions_now(root, [p for p, state in changed.items() if state], by_path, public)
|
|
139
|
+
gone = [f for f in technical if f.rpartition("::")[0] in changed and f not in now]
|
|
140
|
+
doc_registry.observe(conn, root, now, technical, changed, gone)
|
|
141
|
+
features, notes = doc_scope.features(model), []
|
|
142
|
+
candidates, moved = _candidates(model, features, doc_registry.links(conn, root), notes)
|
|
143
|
+
doc_registry.record_links(conn, root, doc_scope.feature_links(model) + candidates)
|
|
144
|
+
claimed = doc_registry.claim(conn, root, MAX_FUNCTIONS, cost=lambda f: 1 + len(features.get(f, ())),
|
|
145
|
+
budget=MAX_NODES)
|
|
146
|
+
if not claimed and not moved:
|
|
147
|
+
return {"claimed": [], "settled": [], "still_pending": [], "review": [], "candidates": [], "prose": None,
|
|
148
|
+
"notes": notes}
|
|
149
|
+
before = {row["fn_id"]: row for row in doc_registry.rows(conn, root) if row["fn_id"] in claimed}
|
|
150
|
+
try:
|
|
151
|
+
doc_scope.generate(root, view_id, model=model)
|
|
152
|
+
except Exception as exc:
|
|
153
|
+
doc_registry.settle(conn, root, {f: (None, f"{type(exc).__name__}: {exc}") for f in claimed})
|
|
154
|
+
raise
|
|
155
|
+
after = {m["id"]: m for m in doc_scope._scan(root)[0] if m["file"].startswith("technical/")}
|
|
156
|
+
current = doc_scope._current_code(root, {f.rpartition("::")[0] for f in claimed}, by_path)
|
|
157
|
+
results = {f: (current[f], None) if f in current and f in after and not doc_scope.changed_parts(after[f], current[f])
|
|
158
|
+
else (None, "the regenerated technical entry still differs from the code") for f in claimed}
|
|
159
|
+
doc_registry.settle(conn, root, results)
|
|
160
|
+
prose = _rewrite_prose(root, model, claimed, features, notes)
|
|
161
|
+
entries = model.entry_points()
|
|
162
|
+
review = [{"fn_id": f, "features": [doc_scope.entry_title(model, e, entries) for e, _h, _o in features.get(f, ())]}
|
|
163
|
+
for f, row in before.items() if row["public"] and row["doc_body_hash"]
|
|
164
|
+
and row["doc_body_hash"] != row["body_hash"] and row["doc_docstring_hash"] == row["docstring_hash"]]
|
|
165
|
+
return {"claimed": claimed, "settled": [f for f, (s, _e) in results.items() if s],
|
|
166
|
+
"still_pending": [f for f, (s, _e) in results.items() if not s], "review": review,
|
|
167
|
+
"candidates": [c for c in candidates if c.get("computed_at") is None], "prose": prose, "notes": notes}
|
|
168
|
+
finally:
|
|
169
|
+
conn.close()
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _candidates(model, features, known, notes):
|
|
173
|
+
"""Embedder candidate rows to keep: those still about an orphan function and an existing feature, plus new ones for
|
|
174
|
+
the orphans that have none. Returns (rows, whether the set shown on the pages changed)."""
|
|
175
|
+
orphans, entries = set(doc_candidates.orphans(model, features)), set(model.entry_points())
|
|
176
|
+
old = [row for row in known if row["origin"] == "embedder"]
|
|
177
|
+
kept = [row for row in old if row["fn_id"] in orphans and row["feature_id"] in entries]
|
|
178
|
+
try:
|
|
179
|
+
new = doc_candidates.propose(model, features, {row["fn_id"] for row in kept})
|
|
180
|
+
except Exception as exc:
|
|
181
|
+
notes.append(f"no feature candidate was proposed ({type(exc).__name__}: {exc}).")
|
|
182
|
+
new = []
|
|
183
|
+
return kept + new, bool(new) or len(kept) != len(old)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _rewrite_prose(root, model, claimed, features, notes):
|
|
187
|
+
"""Rewrite the stale paragraphs of the features the claimed functions belong to, as many features as the node
|
|
188
|
+
budget left by the claimed functions allows. Returns doc_prose.rewrite_stale's result, or None when nothing was
|
|
189
|
+
stale or no Docs model is configured (said in notes)."""
|
|
190
|
+
spent = sum(1 + len(features.get(f, ())) for f in claimed)
|
|
191
|
+
affected = list(dict.fromkeys(entry for f in claimed for entry, _h, _o in features.get(f, ())))
|
|
192
|
+
affected = affected[:max(0, MAX_NODES - spent)]
|
|
193
|
+
pages = doc_scope._read_pages(root, ("functional.md", "simple.md"))
|
|
194
|
+
if not affected or not doc_scope.stale_prose(pages, {a: doc_scope.flow_stamp(model, a) for a in affected}):
|
|
195
|
+
return None
|
|
196
|
+
try:
|
|
197
|
+
result = doc_prose.rewrite_stale(root, model, affected, PROSE_BUDGET_S)
|
|
198
|
+
except ValueError as exc:
|
|
199
|
+
notes.append(str(exc))
|
|
200
|
+
return None
|
|
201
|
+
if result["failed"]:
|
|
202
|
+
notes.append(f"{len(result['failed'])} stale paragraph(s) kept their old text: "
|
|
203
|
+
+ "; ".join(f"{item['block']} in {item['file']}: {item['problem']}" for item in result["failed"][:3])
|
|
204
|
+
+ ".")
|
|
205
|
+
return result
|