@allansantos-dev/smart-tool 0.9.57 → 0.9.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +198 -0
- package/README.md +61 -12
- package/affected_tests.py +1 -1
- package/atomic_io.py +8 -2
- package/code_graph.py +25 -8
- package/code_graph_go.py +526 -12
- package/code_graph_java.mjs +19 -10
- package/code_graph_js.cjs +9 -7
- package/comment_check.py +210 -0
- package/config.py +36 -4
- package/doc_check.py +120 -20
- package/doc_cli.py +239 -0
- package/doc_extras.py +561 -0
- package/doc_insights.py +300 -0
- package/doc_prose.py +371 -0
- package/doc_scope.py +869 -0
- package/doc_validate.py +717 -0
- package/duplicates.py +15 -7
- package/hook_decision.py +62 -14
- package/install.py +59 -1
- package/install_runtime.py +30 -4
- package/layers.py +237 -0
- package/locales/pt-BR.json +10 -1
- package/model_client.py +10 -5
- package/model_defaults.py +9 -6
- package/model_health.py +9 -3
- package/model_upgrade.py +1 -1
- package/package.json +1 -1
- package/pr_impact.py +84 -0
- package/project_insights_ui.py +1 -1
- package/project_maps_ui.py +1 -1
- package/redirect_rule.py +20 -8
- package/setup_ui.py +21 -6
- package/smart_tool_daemon.py +118 -18
- package/usage_meter.py +6 -1
- package/version.py +1 -1
- package/web_adapters/browser/prune_camoufox.py +66 -0
- package/web_fetch.py +1 -1
package/code_graph_js.cjs
CHANGED
|
@@ -65,7 +65,9 @@ const host={
|
|
|
65
65
|
};
|
|
66
66
|
const program=ts.createProgram(names,options,host),checker=program.getTypeChecker();
|
|
67
67
|
const symbols=[],dependencies=[],calls=[],unresolved=[],diagnostics=[],declarations=new Map(),moduleNodes=new Map();
|
|
68
|
-
const maxSymbols
|
|
68
|
+
const {symbols:maxSymbols,edges:maxEdges}=input.limits;
|
|
69
|
+
let dropped=false;
|
|
70
|
+
function room(list,max){if(list.length<max)return true;dropped=true;return false;}
|
|
69
71
|
function filePath(source){return source.fileName.replace('/snapshot/','');}
|
|
70
72
|
function line(source,pos){return source.getLineAndCharacterOfPosition(pos).line+1;}
|
|
71
73
|
function displayName(node) {
|
|
@@ -79,7 +81,7 @@ function displayName(node) {
|
|
|
79
81
|
}
|
|
80
82
|
function isFunction(node){return !!node.body && (ts.isFunctionDeclaration(node)||ts.isMethodDeclaration(node)||ts.isFunctionExpression(node)||ts.isArrowFunction(node)||ts.isConstructorDeclaration(node)||ts.isGetAccessor(node)||ts.isSetAccessor(node));}
|
|
81
83
|
function addSymbol(node,source,owner){
|
|
82
|
-
if(symbols
|
|
84
|
+
if(!room(symbols,maxSymbols))return null;
|
|
83
85
|
let name=displayName(node),parent=node.parent;
|
|
84
86
|
const classParent=parent&&ts.isPropertyDeclaration(parent)?parent.parent:parent;
|
|
85
87
|
const cls=classParent && (ts.isClassDeclaration(classParent)||ts.isClassExpression(classParent)) ? classParent.name?.getText() : null;
|
|
@@ -114,16 +116,16 @@ for(const source of parsed){
|
|
|
114
116
|
const active=declarations.get(node)||owner;
|
|
115
117
|
if((ts.isImportDeclaration(node)||ts.isExportDeclaration(node)) && node.moduleSpecifier && ts.isStringLiteral(node.moduleSpecifier)) {
|
|
116
118
|
const spec=node.moduleSpecifier.text,target=resolve(spec,source.fileName);
|
|
117
|
-
if(dependencies
|
|
119
|
+
if(room(dependencies,maxEdges))dependencies.push({source:filePath(source),target:target?.replace('/snapshot/','')||null,specifier:spec,line:line(source,node.getStart(source)),kind:node.importClause?.isTypeOnly||node.isTypeOnly?'type_import':'import',resolution:target?'resolved':spec.startsWith('.')?'unresolved':'external'});
|
|
118
120
|
}
|
|
119
121
|
if(ts.isCallExpression(node)||ts.isNewExpression(node)){
|
|
120
122
|
const expression=node.expression,at=line(source,node.getStart(source));
|
|
121
123
|
if(ts.isCallExpression(node) && (expression.kind===ts.SyntaxKind.ImportKeyword || ts.isIdentifier(expression)&&expression.text==='require') && node.arguments?.length && ts.isStringLiteral(node.arguments[0])){
|
|
122
124
|
const spec=node.arguments[0].text,target=resolve(spec,source.fileName);
|
|
123
|
-
if(dependencies
|
|
125
|
+
if(room(dependencies,maxEdges))dependencies.push({source:filePath(source),target:target?.replace('/snapshot/','')||null,specifier:spec,line:at,kind:'dynamic_import',resolution:target?'resolved':spec.startsWith('.')?'unresolved':'external'});
|
|
124
126
|
}else{
|
|
125
127
|
const target=targetFor(node);
|
|
126
|
-
if(target && calls
|
|
128
|
+
if(target && room(calls,maxEdges))calls.push({source:(active||moduleSymbol(source)).id,target:target.id,line:at,kind:ts.isNewExpression(node)?'construct':'call',resolution:'static'});
|
|
127
129
|
else if(unresolved.length<1000)unresolved.push({path:filePath(source),line:at,expression:expression.getText(source).slice(0,120),reason:'External or dynamic target, or no resolved implementation in the snapshot.'});
|
|
128
130
|
}
|
|
129
131
|
}
|
|
@@ -142,8 +144,8 @@ for(const source of parsed){
|
|
|
142
144
|
if(components.some(c=>c.template!=null)){
|
|
143
145
|
try{
|
|
144
146
|
const {parseTemplate}=require('./web_adapters/node/node_modules/@angular/compiler/fesm2022/compiler.mjs');
|
|
145
|
-
for(const component of components){if(component.template==null||!component.templatePath)continue;const pathName=component.templatePath.replace('/snapshot/',''),tree=parseTemplate(component.template,pathName);if(tree.errors?.length){diagnostics.push({path:pathName,reason:'Invalid Angular template or syntax not supported in this version.'});continue}templatePaths.add(pathName);const owner={id:pathName+'::'+filePath(component.source)+'::'+component.name+'.template@'+component.inlineLine,path:pathName,name:component.name+'.template',kind:'template',language:'angular',start_line:component.inlineLine,end_line:component.inlineLine+component.template.split('\n').length-1,location_kind:component.templatePath===component.source.fileName?'inline_template':'source_line'};if(symbols
|
|
147
|
+
for(const component of components){if(component.template==null||!component.templatePath)continue;const pathName=component.templatePath.replace('/snapshot/',''),tree=parseTemplate(component.template,pathName);if(tree.errors?.length){diagnostics.push({path:pathName,reason:'Invalid Angular template or syntax not supported in this version.'});continue}templatePaths.add(pathName);const owner={id:pathName+'::'+filePath(component.source)+'::'+component.name+'.template@'+component.inlineLine,path:pathName,name:component.name+'.template',kind:'template',language:'angular',start_line:component.inlineLine,end_line:component.inlineLine+component.template.split('\n').length-1,location_kind:component.templatePath===component.source.fileName?'inline_template':'source_line'};if(!room(symbols,maxSymbols))break;symbols.push(owner);const shadow=new Set(),seen=new WeakSet();function walk(n,fn){if(!n||typeof n!=='object'||seen.has(n))return;seen.add(n);fn(n);for(const [key,value] of Object.entries(n)){if(['sourceSpan','span','startSourceSpan','endSourceSpan','keySpan','valueSpan','parent'].includes(key))continue;if(Array.isArray(value))value.forEach(x=>walk(x,fn));else if(value&&typeof value==='object')walk(value,fn)}}walk(tree.nodes,n=>{if(['Variable','Reference','LetDeclaration'].includes(n.constructor.name)&&n.name)shadow.add(n.name);if(n.constructor.name==='ForLoopBlock'&&n.item?.name)shadow.add(n.item.name)});const seen2=new WeakSet();function inspect(n){if(!n||typeof n!=='object'||seen2.has(n))return;seen2.add(n);const start=n.sourceSpan?.start,at=(typeof start==='number'?component.template.slice(0,start).split('\n').length-1:(start?.line??0))+component.inlineLine;if(n.constructor.name==='Call'||n.constructor.name==='SafeCall'){const receiver=n.receiver;if(receiver?.constructor.name==='PropertyRead'&&['ImplicitReceiver','ThisReceiver'].includes(receiver.receiver?.constructor.name)){const target=shadow.has(receiver.name)?null:component.methods.get(receiver.name);if(target&&room(calls,maxEdges))calls.push({source:owner.id,target:target.id,line:at,kind:'template_call',resolution:'static'});else if(unresolved.length<1000)unresolved.push({path:pathName,line:at,expression:receiver.name+'()',reason:'Unresolved template method, or a local variable that shadows the component.'})}}if(n.constructor.name==='Element'){const matches=components.filter(c=>c.selector===n.name);if(matches.length===1&&filePath(matches[0].source)!==pathName)dependencies.push({source:pathName,target:filePath(matches[0].source),specifier:n.name,line:at,kind:'angular_component',resolution:'resolved'})}if(n.constructor.name==='BindingPipe'){const matches=pipes.filter(p=>p.pipeName===n.name&&p.methods.has('transform'));if(matches.length===1){const pipe=matches[0];dependencies.push({source:pathName,target:filePath(pipe.source),specifier:n.name,line:at,kind:'angular_pipe',resolution:'resolved'});calls.push({source:owner.id,target:pipe.methods.get('transform').id,line:at,kind:'template_pipe',resolution:'static'})}}for(const [key,value] of Object.entries(n)){if(['sourceSpan','span','startSourceSpan','endSourceSpan','keySpan','valueSpan','parent'].includes(key))continue;if(Array.isArray(value))value.forEach(inspect);else if(value&&typeof value==='object')inspect(value)}}inspect(tree.nodes)
|
|
146
148
|
}
|
|
147
149
|
}catch{diagnostics.push({path:'Angular',reason:'The Angular template analyzer did not finish. Check the local runtime.'})}
|
|
148
150
|
}
|
|
149
|
-
process.stdout.write(JSON.stringify({symbols,dependencies:dependencies.slice(0,maxEdges),calls:calls.slice(0,maxEdges),unresolved,diagnostics,parsed:[...parsed.map(filePath),...templatePaths],truncated:
|
|
151
|
+
process.stdout.write(JSON.stringify({symbols,dependencies:dependencies.slice(0,maxEdges),calls:calls.slice(0,maxEdges),unresolved,diagnostics,parsed:[...parsed.map(filePath),...templatePaths],truncated:dropped||calls.length>maxEdges||dependencies.length>maxEdges}));
|
package/comment_check.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Comments in the wrong place, for the edit hook. Documentation belongs in the language's docstring (see doc_check); a
|
|
2
|
+
comment is for a short, non-obvious reason inside a function. Only comments on the lines an edit adds or changes are
|
|
3
|
+
checked, in four forms: commented-out code (the ruff ERA001 / eradicate rule), history or ticket tags, a loose comment
|
|
4
|
+
block above a function instead of its docstring, and an inline block inside a function longer than MAX_INLINE_LINES."""
|
|
5
|
+
import ast
|
|
6
|
+
import io
|
|
7
|
+
import re
|
|
8
|
+
import tokenize
|
|
9
|
+
|
|
10
|
+
import doc_check
|
|
11
|
+
|
|
12
|
+
MAX_INLINE_LINES = 2
|
|
13
|
+
ISSUES = ("commented_code", "history_tag", "loose_docs", "inline_long")
|
|
14
|
+
|
|
15
|
+
_DIRECTIVE = re.compile(
|
|
16
|
+
r"^\s*(?:!|-\*-|type:|noqa|pragma|pylint|fmt:|isort|mypy|pyright|ruff|nosec|go:|\+build|nolint|eslint|@ts-|"
|
|
17
|
+
r"prettier|istanbul|c8 |jshint|biome|#region|#endregion|<reference|SPDX-|Copyright|License)", re.I)
|
|
18
|
+
_DIVIDER = re.compile(r"[-=*_#~]{3,}")
|
|
19
|
+
_TAG = re.compile(
|
|
20
|
+
r"\b[A-Z][A-Z0-9]{1,9}-\d+\b|\b(?:issue|pr|pull request|ticket|pedido|task|bug)\s*#?\d+|\bv\d+(?:\.\d+)+\b"
|
|
21
|
+
r"|\b(?:version|release)\s+\d+(?:\.\d+)+|\b(?:was replaced|were replaced|now uses|changed from|"
|
|
22
|
+
r"added in|fixed in|removed in|new in|formerly|renamed from|before the change|since the rewrite)\b", re.I)
|
|
23
|
+
_BRACED_CODE = re.compile(
|
|
24
|
+
r"^(?:(?:if|for|while|switch|catch)\s*\(.*\)\s*\{?$|(?:if|for|while|switch|else)\b.*\{$|(?:else|try|finally)\s*\{?$|"
|
|
25
|
+
r"(?:return|throw|break|continue)\b[^.]*;$|(?:const|let|var|import|export|package|public|private|protected|static|"
|
|
26
|
+
r"func|type)\b.*[;{]$|(?:[\w<>\[\],.?]+\s+)?[\w.$\[\]]+\s*[-+*/|&]?=(?!=).*;$|[\w.$\[\]]+\s*:=.*|[\w.$]+\([^()]*\)\s*;$|[})]+;?$)")
|
|
27
|
+
_PYTHON_STATEMENTS = (ast.Assign, ast.AugAssign, ast.Return, ast.Raise, ast.Import, ast.ImportFrom, ast.Delete,
|
|
28
|
+
ast.Assert, ast.Pass, ast.Break, ast.Continue, ast.Global, ast.Nonlocal, ast.If, ast.For,
|
|
29
|
+
ast.While, ast.Try, ast.With, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
|
|
30
|
+
_NEXT_DECLARATION = re.compile(r"^\s*(?:@|(?:export\s+)?(?:default\s+)?(?:async\s+)?(?:function|class|def)\b)")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _python_comments(source):
|
|
34
|
+
"""Comment tokens of a Python source: dicts with start, end, text, trailing, doc. Empty when it does not tokenize."""
|
|
35
|
+
found = []
|
|
36
|
+
try:
|
|
37
|
+
for token in tokenize.generate_tokens(io.StringIO(source).readline):
|
|
38
|
+
if token.type == tokenize.COMMENT:
|
|
39
|
+
row, column = token.start
|
|
40
|
+
found.append({"start": row, "end": row, "text": token.string[1:], "trailing": bool(token.line[:column].strip()),
|
|
41
|
+
"doc": False})
|
|
42
|
+
except (tokenize.TokenError, SyntaxError):
|
|
43
|
+
return []
|
|
44
|
+
return found
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _skip_string(source, index, go):
|
|
48
|
+
"""Index just after the string literal opening at `index` (quote, template or raw string, Java text block)."""
|
|
49
|
+
quote = source[index]
|
|
50
|
+
if source.startswith('"""', index):
|
|
51
|
+
end = source.find('"""', index + 3)
|
|
52
|
+
return len(source) if end < 0 else end + 3
|
|
53
|
+
cursor = index + 1
|
|
54
|
+
while cursor < len(source):
|
|
55
|
+
char = source[cursor]
|
|
56
|
+
if char == "\\" and not (go and quote == "`"):
|
|
57
|
+
cursor += 2
|
|
58
|
+
continue
|
|
59
|
+
if char == quote:
|
|
60
|
+
return cursor + 1
|
|
61
|
+
if char == "\n" and quote != "`":
|
|
62
|
+
return cursor
|
|
63
|
+
cursor += 1
|
|
64
|
+
return cursor
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _braced_comments(source, go):
|
|
68
|
+
"""Comments of a JavaScript, TypeScript, Java or Go source, skipping what is inside string literals."""
|
|
69
|
+
found, index, line, line_start = [], 0, 1, 0
|
|
70
|
+
while index < len(source):
|
|
71
|
+
char = source[index]
|
|
72
|
+
if char == "\n":
|
|
73
|
+
line, index, line_start = line + 1, index + 1, index + 1
|
|
74
|
+
elif source.startswith("//", index):
|
|
75
|
+
end = source.find("\n", index)
|
|
76
|
+
end = len(source) if end < 0 else end
|
|
77
|
+
found.append({"start": line, "end": line, "text": source[index + 2:end],
|
|
78
|
+
"trailing": bool(source[line_start:index].strip()), "doc": False})
|
|
79
|
+
index = end
|
|
80
|
+
elif source.startswith("/*", index):
|
|
81
|
+
end = source.find("*/", index + 2)
|
|
82
|
+
end = len(source) if end < 0 else end + 2
|
|
83
|
+
body = source[index:end]
|
|
84
|
+
lines = body.count("\n")
|
|
85
|
+
found.append({"start": line, "end": line + lines, "text": body[2:-2] if body.endswith("*/") else body[2:],
|
|
86
|
+
"trailing": bool(source[line_start:index].strip()),
|
|
87
|
+
"doc": body.startswith("/**") and not body.startswith("/**/")})
|
|
88
|
+
line += lines
|
|
89
|
+
index = end
|
|
90
|
+
elif char in "\"'`":
|
|
91
|
+
end = _skip_string(source, index, go)
|
|
92
|
+
line += source.count("\n", index, end)
|
|
93
|
+
index = end
|
|
94
|
+
else:
|
|
95
|
+
index += 1
|
|
96
|
+
return found
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _blocks(comments):
|
|
100
|
+
"""Consecutive standalone line comments merged into one block; directives and dividers dropped."""
|
|
101
|
+
blocks = []
|
|
102
|
+
for comment in comments:
|
|
103
|
+
text = comment["text"]
|
|
104
|
+
if comment["doc"] or _DIRECTIVE.match(text) or _DIVIDER.search(text):
|
|
105
|
+
continue
|
|
106
|
+
last = blocks[-1] if blocks else None
|
|
107
|
+
if (last and not last["trailing"] and not comment["trailing"] and last["end"] + 1 == comment["start"]
|
|
108
|
+
and last["end"] == last["start"] + len(last["lines"]) - 1):
|
|
109
|
+
last["end"] = comment["end"]
|
|
110
|
+
last["lines"].append(text.strip())
|
|
111
|
+
else:
|
|
112
|
+
blocks.append({"start": comment["start"], "end": comment["end"], "trailing": comment["trailing"],
|
|
113
|
+
"lines": [text.strip()] if comment["end"] == comment["start"] else
|
|
114
|
+
[part.strip(" *") for part in text.splitlines()]})
|
|
115
|
+
return blocks
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _python_code(text):
|
|
119
|
+
"""True when the comment text is a Python statement and not prose (ERA001 heuristic)."""
|
|
120
|
+
text = text.strip()
|
|
121
|
+
if len(text.split()) < 2 and "(" not in text and "=" not in text:
|
|
122
|
+
return False
|
|
123
|
+
for candidate in (text, text + " pass" if text.endswith(":") else None):
|
|
124
|
+
if candidate is None:
|
|
125
|
+
continue
|
|
126
|
+
try:
|
|
127
|
+
body = ast.parse(candidate).body
|
|
128
|
+
except (SyntaxError, ValueError):
|
|
129
|
+
continue
|
|
130
|
+
if body and all(_python_statement(node, text) for node in body):
|
|
131
|
+
return True
|
|
132
|
+
return False
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _python_statement(node, text):
|
|
136
|
+
"""Whether a parsed node of a comment reads as code: bare keywords like `return value is cached` need punctuation."""
|
|
137
|
+
if isinstance(node, (ast.Return, ast.Raise, ast.Delete, ast.Assert, ast.Global, ast.Nonlocal)):
|
|
138
|
+
return any(char in text for char in "()[]{}=.,'\"")
|
|
139
|
+
if isinstance(node, ast.AnnAssign):
|
|
140
|
+
return node.value is not None
|
|
141
|
+
if isinstance(node, ast.Expr):
|
|
142
|
+
return isinstance(node.value, (ast.Call, ast.Await))
|
|
143
|
+
return isinstance(node, _PYTHON_STATEMENTS)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _is_code(text, python):
|
|
147
|
+
return _python_code(text) if python else bool(_BRACED_CODE.match(text.strip()))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _inside(functions, block, lines, go):
|
|
151
|
+
"""The function whose body holds the block, or None."""
|
|
152
|
+
for function in functions:
|
|
153
|
+
if go and not lines[function["start"] - 1].lstrip().startswith("func"):
|
|
154
|
+
continue
|
|
155
|
+
if function["start"] < block["start"] and block["end"] <= function["end"]:
|
|
156
|
+
return function
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _next_line(lines, end):
|
|
161
|
+
"""Number and text of the first non-blank line after line `end` (1-based), or (0, "")."""
|
|
162
|
+
for number in range(end + 1, len(lines) + 1):
|
|
163
|
+
if lines[number - 1].strip():
|
|
164
|
+
return number, lines[number - 1]
|
|
165
|
+
return 0, ""
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def review(path, before, after, edited=None):
|
|
169
|
+
"""Comments the edit from before to after leaves in a wrong place.
|
|
170
|
+
|
|
171
|
+
Parameters: path (str) edited file; before (str or None) text before the edit; after (str) text after it; edited
|
|
172
|
+
is `doc_check.written` of the same edit when the caller already has it.
|
|
173
|
+
Returns: list of dicts with line, issue (one of ISSUES), name (enclosing function or None) and text, in line order;
|
|
174
|
+
empty for an unsupported file or one that does not tokenize.
|
|
175
|
+
"""
|
|
176
|
+
if not doc_check.supported(path):
|
|
177
|
+
return []
|
|
178
|
+
python = path.lower().endswith(doc_check.PYTHON)
|
|
179
|
+
go = path.lower().endswith(doc_check.GO)
|
|
180
|
+
comments = _python_comments(after) if python else _braced_comments(after, go)
|
|
181
|
+
if not comments:
|
|
182
|
+
return []
|
|
183
|
+
edited = edited or doc_check.written(path, before, after)
|
|
184
|
+
touched = doc_check._touched_lines(before, after)
|
|
185
|
+
lines = after.splitlines()
|
|
186
|
+
declarations = {function["start"] for function in edited["all"]}
|
|
187
|
+
findings = []
|
|
188
|
+
for block in _blocks(comments):
|
|
189
|
+
if not touched.intersection(range(block["start"], block["end"] + 1)):
|
|
190
|
+
continue
|
|
191
|
+
text = " ".join(block["lines"])
|
|
192
|
+
owner = _inside(edited["all"], block, lines, go)
|
|
193
|
+
name = owner["name"] if owner else None
|
|
194
|
+
issue = None
|
|
195
|
+
if _is_code(block["lines"][0], python) or len(block["lines"]) > 1 and _is_code(" ".join(block["lines"]), python):
|
|
196
|
+
issue = "commented_code"
|
|
197
|
+
elif _TAG.search(text):
|
|
198
|
+
issue = "history_tag"
|
|
199
|
+
elif not block["trailing"] and not go and _loose_docs(lines, block, declarations):
|
|
200
|
+
issue = "loose_docs"
|
|
201
|
+
elif owner and block["end"] - block["start"] + 1 > MAX_INLINE_LINES:
|
|
202
|
+
issue = "inline_long"
|
|
203
|
+
if issue:
|
|
204
|
+
findings.append({"line": block["start"], "issue": issue, "name": name, "text": text[:80]})
|
|
205
|
+
return findings
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _loose_docs(lines, block, declarations):
|
|
209
|
+
number, text = _next_line(lines, block["end"])
|
|
210
|
+
return bool(number) and (number in declarations or bool(_NEXT_DECLARATION.match(text)))
|
package/config.py
CHANGED
|
@@ -21,7 +21,13 @@ CONFIG_PATH = os.path.join(CONFIG_DIR, "config.json")
|
|
|
21
21
|
_LOCK = threading.RLock()
|
|
22
22
|
|
|
23
23
|
# The config fields that select a gateway model; every per-role table (recommended models, failure log) keys on these.
|
|
24
|
-
MODEL_ROLES = ("router_model", "scope_model", "scout_model", "research_model", "
|
|
24
|
+
MODEL_ROLES = ("router_model", "scope_model", "scout_model", "research_model", "docs_model", "embedding_model",
|
|
25
|
+
"rerank_model")
|
|
26
|
+
|
|
27
|
+
# Extra documents written by `project_manage action=docs generate=true` beside docs/scope's three levels; each key is a
|
|
28
|
+
# document and true (the default) writes it, false skips it and removes the file Smart Tool wrote earlier.
|
|
29
|
+
DOCS_EXTRAS = ("structure", "coverage", "modules", "tests", "configuration", "dependencies", "adr", "flows",
|
|
30
|
+
"features_tests", "hotspots", "coupling", "dead_code", "tour", "glossary", "index")
|
|
25
31
|
|
|
26
32
|
DEFAULT_CONFIG = {
|
|
27
33
|
"router_model": "",
|
|
@@ -30,14 +36,17 @@ DEFAULT_CONFIG = {
|
|
|
30
36
|
"embedding_model": "",
|
|
31
37
|
"rerank_model": "",
|
|
32
38
|
"research_model": "",
|
|
39
|
+
"docs_model": "",
|
|
33
40
|
"research_max_iterations": 3,
|
|
34
41
|
"quick_search_timeout_s": 120,
|
|
35
42
|
"site_memory_similarity_threshold": 0.35,
|
|
36
43
|
"local_models": "fallback",
|
|
37
44
|
"hook_mode": "redirect",
|
|
38
|
-
"doc_mode": "
|
|
45
|
+
"doc_mode": "require",
|
|
46
|
+
"comment_mode": "require",
|
|
39
47
|
"duplicate_mode": "warn",
|
|
40
48
|
"dependency_mode": "warn",
|
|
49
|
+
"docs_extras": {key: True for key in DOCS_EXTRAS},
|
|
41
50
|
"auto_update": True,
|
|
42
51
|
"language": "",
|
|
43
52
|
"pattern_proposals": "on",
|
|
@@ -60,6 +69,10 @@ HOOK_MODES = ("redirect", "advise", "off")
|
|
|
60
69
|
# Edit hook on functions without a docstring: require = deny the edit until it adds one; remind = let it run and tell
|
|
61
70
|
# the agent; off = say nothing. Independent of hook_mode, which only routes searches and reads.
|
|
62
71
|
DOC_MODES = ("require", "remind", "off")
|
|
72
|
+
# Edit hook on comments an edit adds in the wrong place (commented-out code, ticket or history tags, a loose block above
|
|
73
|
+
# a function instead of its docstring, a long block inside a body): require = deny the edit until they are gone;
|
|
74
|
+
# remind = let it run and tell the agent; off = say nothing. A short comment explaining a non-obvious reason is allowed.
|
|
75
|
+
COMMENT_MODES = ("require", "remind", "off")
|
|
63
76
|
# Edit hook on functions that copy one already indexed (exact or near-identical body): warn = let the edit run and
|
|
64
77
|
# point to the existing function; off = say nothing. Never blocks: near matches are leads, not defects.
|
|
65
78
|
DUPLICATE_MODES = ("warn", "off")
|
|
@@ -71,8 +84,9 @@ DEPENDENCY_MODES = ("warn", "off")
|
|
|
71
84
|
PATTERN_PROPOSAL_MODES = ("on", "off")
|
|
72
85
|
# Structured output of the model calls that must return JSON (research, scout, freshness): response_format = the
|
|
73
86
|
# OpenAI-style `response_format` json_schema (applied on every gateway route); output_config = Anthropic `output_config`
|
|
74
|
-
# in the chat body (also reports truncation as finish_reason "length"); auto = output_config for the models whose route
|
|
75
|
-
# (the id prefix before the first dot) is in `output_config_routes`, response_format for all others
|
|
87
|
+
# in the chat body (also reports truncation as finish_reason "length"); auto = output_config for the Claude models whose route
|
|
88
|
+
# (the id prefix before the first dot) is in `output_config_routes`, response_format for all others (Gemini on
|
|
89
|
+
# vertex_ai ignores output_config and answers loose text).
|
|
76
90
|
STRUCTURED_OUTPUT_MODES = ("auto", "response_format", "output_config")
|
|
77
91
|
|
|
78
92
|
|
|
@@ -83,6 +97,13 @@ def doc_mode(cfg=None):
|
|
|
83
97
|
return value
|
|
84
98
|
|
|
85
99
|
|
|
100
|
+
def comment_mode(cfg=None):
|
|
101
|
+
value = (cfg if cfg is not None else load_config()).get("comment_mode") or DEFAULT_CONFIG["comment_mode"]
|
|
102
|
+
if value not in COMMENT_MODES:
|
|
103
|
+
raise ValueError(f"Invalid comment_mode ({value!r}) in {CONFIG_PATH}: use require, remind or off.")
|
|
104
|
+
return value
|
|
105
|
+
|
|
106
|
+
|
|
86
107
|
def duplicate_mode(cfg=None):
|
|
87
108
|
value = (cfg if cfg is not None else load_config()).get("duplicate_mode") or DEFAULT_CONFIG["duplicate_mode"]
|
|
88
109
|
if value not in DUPLICATE_MODES:
|
|
@@ -97,6 +118,17 @@ def dependency_mode(cfg=None):
|
|
|
97
118
|
return value
|
|
98
119
|
|
|
99
120
|
|
|
121
|
+
def docs_extras(cfg=None):
|
|
122
|
+
"""Which extra documents the docs generator writes: every key of DOCS_EXTRAS, true unless the stored `docs_extras`
|
|
123
|
+
object sets it to false; an unknown key or a value that is not true or false fails loudly."""
|
|
124
|
+
stored = (cfg if cfg is not None else load_config()).get("docs_extras")
|
|
125
|
+
if stored is None:
|
|
126
|
+
stored = {}
|
|
127
|
+
if not isinstance(stored, dict) or set(stored) - set(DOCS_EXTRAS) or any(type(v) is not bool for v in stored.values()):
|
|
128
|
+
raise ValueError(f"Invalid docs_extras in {CONFIG_PATH}: an object with only {', '.join(DOCS_EXTRAS)}, each true or false.")
|
|
129
|
+
return {**{key: True for key in DOCS_EXTRAS}, **stored}
|
|
130
|
+
|
|
131
|
+
|
|
100
132
|
def pattern_proposals(cfg=None):
|
|
101
133
|
"""on: the router model reviews searches the redirect rule let run and proposes block patterns for the user to
|
|
102
134
|
accept; off: no review."""
|
package/doc_check.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"""Function docstrings: which functions a file has, whether each one is documented, and which ones an edit touched.
|
|
2
2
|
Python is read with ast; TypeScript/JavaScript and Java with the code graph analyzers, where the documentation is the
|
|
3
|
-
/** */ block right above the function
|
|
4
|
-
|
|
3
|
+
/** */ block right above the function; Go with the Go reader of the code graph, where the documentation is the doc
|
|
4
|
+
comment of each exported declaration (functions, methods, types, consts and vars) and must start with its name. Used by
|
|
5
|
+
the edit hook (missing or possibly stale docstrings), the docstring coverage listing and the one-line summaries in
|
|
6
|
+
graph and impact results."""
|
|
5
7
|
import ast
|
|
6
8
|
import difflib
|
|
7
9
|
import hashlib
|
|
@@ -11,13 +13,18 @@ import threading
|
|
|
11
13
|
from collections import Counter, OrderedDict, defaultdict
|
|
12
14
|
|
|
13
15
|
import code_graph
|
|
16
|
+
import code_graph_go
|
|
14
17
|
import index_profile
|
|
15
18
|
import index_scope
|
|
16
19
|
|
|
17
20
|
PYTHON = (".py",)
|
|
18
21
|
SCRIPT = (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts")
|
|
19
22
|
JAVA = (".java",)
|
|
20
|
-
|
|
23
|
+
GO = (".go",)
|
|
24
|
+
SUPPORTED = PYTHON + SCRIPT + JAVA + GO
|
|
25
|
+
GO_COMMON_METHODS = {"Error", "Read", "ServeHTTP", "String", "Write", "Unwrap"}
|
|
26
|
+
GO_SORT_METHODS = {"Len", "Less", "Swap"}
|
|
27
|
+
GO_ARTICLES = ("A ", "An ", "The ", "This ")
|
|
21
28
|
MAX_PARSED = 64
|
|
22
29
|
_SPACE = re.compile(r"\s+")
|
|
23
30
|
_PARSED = OrderedDict()
|
|
@@ -41,25 +48,47 @@ def _python_functions(source):
|
|
|
41
48
|
function) and PEP 698: an @override method inherits its documentation."""
|
|
42
49
|
tree = ast.parse(source)
|
|
43
50
|
found = []
|
|
51
|
+
imported = _imported_names(tree)
|
|
44
52
|
|
|
45
|
-
def visit(node, owner, public, nested):
|
|
53
|
+
def visit(node, owner, public, nested, bases=()):
|
|
46
54
|
for child in ast.iter_child_nodes(node):
|
|
47
55
|
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
48
56
|
doc = ast.get_docstring(child)
|
|
49
57
|
returns = f" -> {ast.unparse(child.returns)}" if child.returns else ""
|
|
50
58
|
found.append({"name": f"{owner}{child.name}", "start": child.lineno, "end": child.end_lineno,
|
|
51
59
|
"documented": doc is not None, "doc": _first_line(doc or ""),
|
|
60
|
+
"decorators": [ast.unparse(d) for d in child.decorator_list], "bases": list(bases),
|
|
52
61
|
"signature": f"{child.name}({ast.unparse(child.args)}){returns}",
|
|
53
62
|
"required": public and not nested and _public_name(child.name)
|
|
54
63
|
and not any(ast.unparse(d).split(".")[-1] == "override"
|
|
55
64
|
for d in child.decorator_list)})
|
|
56
65
|
visit(child, f"{owner}{child.name}.", False, True)
|
|
57
66
|
elif isinstance(child, ast.ClassDef):
|
|
58
|
-
|
|
67
|
+
bases = [_dotted(ast.unparse(b), imported) for b in child.bases]
|
|
68
|
+
visit(child, f"{owner}{child.name}.", public and _public_name(child.name), nested, bases)
|
|
59
69
|
visit(tree, "", True, False)
|
|
60
70
|
return found
|
|
61
71
|
|
|
62
72
|
|
|
73
|
+
def _imported_names(tree):
|
|
74
|
+
"""Local name -> dotted origin for the imports of a module; a relative import keeps its leading dots."""
|
|
75
|
+
names = {}
|
|
76
|
+
for node in ast.walk(tree):
|
|
77
|
+
if isinstance(node, ast.Import):
|
|
78
|
+
for alias in node.names:
|
|
79
|
+
names[alias.asname or alias.name.split(".")[0]] = alias.name if alias.asname else alias.name.split(".")[0]
|
|
80
|
+
elif isinstance(node, ast.ImportFrom):
|
|
81
|
+
for alias in node.names:
|
|
82
|
+
names[alias.asname or alias.name] = f"{'.' * node.level}{node.module or ''}.{alias.name}"
|
|
83
|
+
return names
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _dotted(expression, imported):
|
|
87
|
+
"""Base-class expression with its first name replaced by the module it was imported from."""
|
|
88
|
+
head, _dot, rest = expression.partition(".")
|
|
89
|
+
return imported[head] + (f".{rest}" if rest else "") if head in imported else expression
|
|
90
|
+
|
|
91
|
+
|
|
63
92
|
def _overload_head(lines, index, name):
|
|
64
93
|
"""Line where the overload signature ending at index starts, or None when the lines above are not an overload of
|
|
65
94
|
name (a declaration ending in ';')."""
|
|
@@ -131,7 +160,7 @@ def _required(path, symbol, parent, first_line, annotations):
|
|
|
131
160
|
|
|
132
161
|
|
|
133
162
|
def _braced_functions(path, source, symbols):
|
|
134
|
-
lines = source.
|
|
163
|
+
lines = source.split("\n")
|
|
135
164
|
by_id = {s.get("id"): s for s in symbols}
|
|
136
165
|
script = path.lower().endswith(SCRIPT)
|
|
137
166
|
found = []
|
|
@@ -150,6 +179,65 @@ def _braced_functions(path, source, symbols):
|
|
|
150
179
|
return found
|
|
151
180
|
|
|
152
181
|
|
|
182
|
+
def _go_form(doc, name, articles):
|
|
183
|
+
"""Whether a Go doc comment starts with the declared name, after an article for a type (revive exported rule)."""
|
|
184
|
+
first = next((line.strip() for line in doc.splitlines() if line.strip()), "")
|
|
185
|
+
if articles:
|
|
186
|
+
first = next((first[len(a):] for a in GO_ARTICLES if first.startswith(a)), first)
|
|
187
|
+
return first.startswith(name + " ")
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _go_functions(path, source):
|
|
191
|
+
"""Exported Go declarations as the revive exported rule checks them: "required" outside test files and package
|
|
192
|
+
main, for exported functions, types, consts and vars and the exported methods of exported types (Error, String,
|
|
193
|
+
ServeHTTP and the other common interface methods, and Len/Less/Swap of a sortable type, are left out). A doc
|
|
194
|
+
comment that does not start with the name counts as undocumented ("malformed"). A parenthesized const or var
|
|
195
|
+
block is satisfied by its own doc comment; without one, its first exported spec lacking a comment stands for the
|
|
196
|
+
block, as revive reports it once."""
|
|
197
|
+
package, decls = code_graph_go.declarations(source)
|
|
198
|
+
importable = not path.lower().endswith("_test.go") and package != "main"
|
|
199
|
+
methods = defaultdict(set)
|
|
200
|
+
for decl in decls:
|
|
201
|
+
if decl["kind"] == "method":
|
|
202
|
+
methods[decl["receiver"]].add(decl["name"])
|
|
203
|
+
sortable = {receiver for receiver, names in methods.items() if GO_SORT_METHODS <= names}
|
|
204
|
+
found, reported_blocks = [], set()
|
|
205
|
+
|
|
206
|
+
def add(decl, name, short, required, doc, articles=False):
|
|
207
|
+
documented = bool(doc) and _go_form(doc, short, articles)
|
|
208
|
+
found.append({"name": name, "start": decl["start"], "end": decl["end"], "documented": documented,
|
|
209
|
+
"doc": _first_line(doc or ""), "signature": decl["signature"], "required": required,
|
|
210
|
+
"kind": decl["kind"], "convention": "go", "malformed": bool(doc) and not documented})
|
|
211
|
+
|
|
212
|
+
for decl in decls:
|
|
213
|
+
short = decl["name"]
|
|
214
|
+
if decl["kind"] in ("function", "method"):
|
|
215
|
+
receiver = decl["receiver"]
|
|
216
|
+
required = importable and short[:1].isupper() and (
|
|
217
|
+
not receiver or receiver[:1].isupper() and short not in GO_COMMON_METHODS
|
|
218
|
+
and not (short in GO_SORT_METHODS and receiver in sortable))
|
|
219
|
+
add(decl, f"{receiver}.{short}" if receiver else short, short, required, decl["doc"])
|
|
220
|
+
elif decl["kind"] in ("type", "interface"):
|
|
221
|
+
add(decl, short, short, importable and short[:1].isupper(), decl["doc"] or decl["group_doc"], True)
|
|
222
|
+
else:
|
|
223
|
+
exported = next((n for n in decl["names"] if n[:1].isupper()), None)
|
|
224
|
+
if exported is None:
|
|
225
|
+
continue
|
|
226
|
+
if decl["grouped"] and decl["group_doc"]:
|
|
227
|
+
if decl["block"] not in reported_blocks:
|
|
228
|
+
reported_blocks.add(decl["block"])
|
|
229
|
+
found.append({"name": exported, "start": decl["start"], "end": decl["end"], "documented": True,
|
|
230
|
+
"doc": _first_line(decl["group_doc"]), "signature": "", "required": importable,
|
|
231
|
+
"kind": decl["kind"], "convention": "go", "malformed": False})
|
|
232
|
+
continue
|
|
233
|
+
if decl["grouped"] and not decl["doc"]:
|
|
234
|
+
if decl["block"] in reported_blocks:
|
|
235
|
+
continue
|
|
236
|
+
reported_blocks.add(decl["block"])
|
|
237
|
+
add(decl, exported, exported, importable, decl["doc"])
|
|
238
|
+
return found
|
|
239
|
+
|
|
240
|
+
|
|
153
241
|
def functions(path, source, symbols=None):
|
|
154
242
|
"""Functions of one file with start/end lines, documented flag, first doc line and normalized signature. symbols
|
|
155
243
|
are the code graph symbols of that file when already known; otherwise the analyzer runs on source."""
|
|
@@ -159,29 +247,40 @@ def functions(path, source, symbols=None):
|
|
|
159
247
|
return _python_functions(source)
|
|
160
248
|
except SyntaxError:
|
|
161
249
|
return []
|
|
250
|
+
if lower.endswith(GO):
|
|
251
|
+
return _go_functions(path, source)
|
|
162
252
|
if symbols is None:
|
|
163
253
|
symbols = analyze(path, source)[0]
|
|
164
254
|
return _braced_functions(path, source, symbols)
|
|
165
255
|
|
|
166
256
|
|
|
167
257
|
def analyze(path, *sources):
|
|
168
|
-
"""Analyzer symbols of several versions of one TypeScript/JavaScript/Java file
|
|
169
|
-
|
|
258
|
+
"""Analyzer symbols of several versions of one TypeScript/JavaScript/Java file (an edit hook needs the file before
|
|
259
|
+
and after; the next edit's 'before' is this 'after'). See analyze_files."""
|
|
260
|
+
return analyze_files([(path, source) for source in sources])
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def analyze_files(items):
|
|
264
|
+
"""Analyzer symbols of each (path, source) pair, in order, from one analyzer run per language for the pairs not
|
|
265
|
+
seen yet: starting the analyzer costs more than reading a file, so callers with many files pass them together.
|
|
170
266
|
An analysis that failed or timed out is returned empty but not cached, so the next call tries again."""
|
|
171
|
-
|
|
172
|
-
keys = [hashlib.sha1(f"{name}\0{source}".encode("utf-8")).hexdigest() for source in sources]
|
|
267
|
+
keys = [hashlib.sha1(f"{os.path.basename(path)}\0{source}".encode("utf-8")).hexdigest() for path, source in items]
|
|
173
268
|
with _PARSED_LOCK:
|
|
174
|
-
missing = {key:
|
|
269
|
+
missing = {key: item for key, item in zip(keys, items) if key not in _PARSED}
|
|
175
270
|
found = {}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
271
|
+
for extensions, analyzer in ((SCRIPT, code_graph._javascript_graph), (JAVA, code_graph._java_graph)):
|
|
272
|
+
group = [(key, path, source) for key, (path, source) in missing.items() if path.lower().endswith(extensions)]
|
|
273
|
+
if not group:
|
|
274
|
+
continue
|
|
275
|
+
batch = {f"v{i}/{os.path.basename(path)}": source for i, (_key, path, source) in enumerate(group)}
|
|
276
|
+
analysis = analyzer(batch)
|
|
277
|
+
names = list(batch)
|
|
278
|
+
result = {key: [dict(s, path=os.path.basename(path)) for s in analysis.get("symbols") or []
|
|
279
|
+
if s.get("path") == names[i]] for i, (key, path, _source) in enumerate(group)}
|
|
280
|
+
found.update(result)
|
|
182
281
|
with _PARSED_LOCK:
|
|
183
282
|
if not analysis.get("retryable"):
|
|
184
|
-
_PARSED.update(
|
|
283
|
+
_PARSED.update(result)
|
|
185
284
|
while len(_PARSED) > MAX_PARSED:
|
|
186
285
|
_PARSED.popitem(last=False)
|
|
187
286
|
with _PARSED_LOCK:
|
|
@@ -190,7 +289,7 @@ def analyze(path, *sources):
|
|
|
190
289
|
|
|
191
290
|
|
|
192
291
|
def _touched_lines(before, after):
|
|
193
|
-
old, new = (before or "").
|
|
292
|
+
old, new = (before or "").split("\n"), after.split("\n")
|
|
194
293
|
touched = set()
|
|
195
294
|
for tag, _i1, _i2, j1, j2 in difflib.SequenceMatcher(None, old, new, autojunk=False).get_opcodes():
|
|
196
295
|
if tag != "equal":
|
|
@@ -249,7 +348,8 @@ def coverage(root, include_tests=False, limit=30, view_id=None):
|
|
|
249
348
|
continue
|
|
250
349
|
found = [f for f in functions(path, source, by_path[path]) if f["required"] or f["documented"]]
|
|
251
350
|
total += len(found)
|
|
252
|
-
missing += [{"path": path, "function": f["name"], "line": f["start"]
|
|
351
|
+
missing += [{"path": path, "function": f["name"], "line": f["start"], **({"malformed": True} if f.get("malformed")
|
|
352
|
+
else {})} for f in found if not f["documented"]]
|
|
253
353
|
per_file = Counter(item["path"] for item in missing)
|
|
254
354
|
return {"functions": total, "documented": total - len(missing),
|
|
255
355
|
"coverage_percent": round(100 * (total - len(missing)) / total, 1) if total else None,
|