@allansantos-dev/smart-tool 0.9.78 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -0
- package/README.md +65 -18
- package/affected_tests.py +4 -0
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_graph.py +39 -17
- package/code_graph_go.py +6 -6
- package/code_graph_java.mjs +10 -9
- package/code_graph_js.cjs +8 -8
- package/code_health.py +180 -0
- package/code_impact.py +16 -2
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/doc_scope.py +1 -1
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/duplicates.py +1 -1
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/layers.py +3 -3
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +113 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/code_graph_java.mjs
CHANGED
|
@@ -10,15 +10,15 @@ if(nodeModule.registerHooks){const bundle=new URL('chevrotain/lib/chevrotain.mjs
|
|
|
10
10
|
const {parse}=await import(new URL('java-parser/src/index.js',modules).href);
|
|
11
11
|
const input=JSON.parse(fs.readFileSync(0,'utf8'));
|
|
12
12
|
const symbols=[],dependencies=[],calls=[],unresolved=[],diagnostics=[],units=[],classes=new Map(),methodNodes=new Map(),kids=new WeakMap();
|
|
13
|
-
const {symbols:MAX_SYMBOLS,edges:MAX_EDGES}=input.limits;
|
|
14
|
-
function room(list,max){if(list.length<max)return true;
|
|
13
|
+
const {symbols:MAX_SYMBOLS,edges:MAX_EDGES}=input.limits;const leftOut={symbols:0,edges:0};
|
|
14
|
+
function room(list,max,kind){if(list.length<max)return true;leftOut[kind]++;return false}
|
|
15
15
|
const NO_CHILDREN=Object.freeze([]);
|
|
16
16
|
function children(n){const groups=n?.children;if(!groups)return NO_CHILDREN;let list=kids.get(n);if(list)return list;list=[];for(const key in groups)for(const child of groups[key])if(child.children)list.push(child);kids.set(n,list);return list}
|
|
17
17
|
function all(n,name,stop=false){const result=[];function visit(node){if(node?.name===name){result.push(node);if(stop)return}for(const child of children(node))visit(child)}visit(n);return result}
|
|
18
18
|
function first(n,name){return all(n,name,true)[0]}
|
|
19
19
|
function identifiers(n){const out=[];function visit(node){if(node?.children)for(const [key,items] of Object.entries(node.children))for(const child of items){if(child.children)visit(child);else if(key==='Identifier')out.push(child)}}visit(n);return out.sort((a,b)=>a.startOffset-b.startOffset)}
|
|
20
20
|
function text(u,n){return n?.location?u.text.slice(n.location.startOffset,n.location.endOffset+1):''}
|
|
21
|
-
function addSymbol(u,n,name,kind){if(!room(symbols,MAX_SYMBOLS))return null;const s={id:u.path+'::'+name+'@'+n.location.startLine+':'+n.location.startOffset,path:u.path,name,kind,language:'java',start_line:n.location.startLine,end_line:n.location.endLine};symbols.push(s);return s}
|
|
21
|
+
function addSymbol(u,n,name,kind){if(!room(symbols,MAX_SYMBOLS,'symbols'))return null;const s={id:u.path+'::'+name+'@'+n.location.startLine+':'+n.location.startOffset,path:u.path,name,kind,language:'java',start_line:n.location.startLine,end_line:n.location.endLine};symbols.push(s);return s}
|
|
22
22
|
const classKinds=new Set(['normalClassDeclaration','normalInterfaceDeclaration','enumDeclaration','recordDeclaration']);
|
|
23
23
|
for(const file of input.files){try{const tree=parse(file.text),pkg=first(tree,'packageDeclaration'),unit={...file,tree,package:pkg?identifiers(pkg).map(t=>t.image).join('.'): '',imports:[],classes:[]};
|
|
24
24
|
for(const n of all(tree,'importDeclaration')){const raw=text(unit,n).replace(/^import\s+/,'').replace(/;\s*$/,'').trim(),stat=raw.startsWith('static '),spec=(stat?raw.slice(7):raw).replace(/\s+/g,'');unit.imports.push({spec,static:stat,line:n.location.startLine})}
|
|
@@ -38,18 +38,18 @@ for(const unit of units){
|
|
|
38
38
|
function walk(n,cls=null,method=null,vars=new Map(),owner=null){if(classKinds.has(n.name))cls=unit.classes.find(c=>c.node===n)||cls;if(cls&&n.name==='classBody'&&n!==cls.body)return;
|
|
39
39
|
if(methodNodes.has(n)){method=methodNodes.get(n);vars=new Map(cls.fields);collectVariables(unit,n,vars);owner=method.symbol}
|
|
40
40
|
if(n.name==='lambdaExpression'){vars=new Map(vars);for(const param of identifiers(n.children.lambdaParameters?.[0]))vars.set(param.image,null);owner=addSymbol(unit,n,(cls?.local||'')+'.lambda@'+n.location.startLine,'function')||owner}
|
|
41
|
-
if(cls&&['unannType','classType','interfaceType'].includes(n.name)){const type=resolveType(unit,text(unit,n),cls);if(type&&type.unit.path!==unit.path&&room(dependencies,MAX_EDGES))dependencies.push({source:unit.path,target:type.unit.path,specifier:type.fqn,line:n.location.startLine,kind:'java_type',resolution:'resolved'})}
|
|
41
|
+
if(cls&&['unannType','classType','interfaceType'].includes(n.name)){const type=resolveType(unit,text(unit,n),cls);if(type&&type.unit.path!==unit.path&&room(dependencies,MAX_EDGES,'edges'))dependencies.push({source:unit.path,target:type.unit.path,specifier:type.fqn,line:n.location.startLine,kind:'java_type',resolution:'resolved'})}
|
|
42
42
|
if(cls&&n.name==='primary'){
|
|
43
43
|
const prefix=n.children.primaryPrefix?.[0],prefixText=text(unit,prefix).trim(),suffixes=n.children.primarySuffix||[];let chain=prefixText,fluent=false;
|
|
44
44
|
const newMatch=/^new\s+([\w.$]+)/.exec(prefixText);let constructed=newMatch?resolveType(unit,newMatch[1],cls):null;
|
|
45
|
-
const link=(type,line)=>{if(type&&type.unit.path!==unit.path&&room(dependencies,MAX_EDGES))dependencies.push({source:unit.path,target:type.unit.path,specifier:type.fqn,line,kind:'java_type',resolution:'resolved'})};
|
|
46
|
-
if(constructed&&room(calls,MAX_EDGES))calls.push({source:(owner||cls.symbol).id,target:constructed.symbol.id,line:n.location.startLine,kind:'construct',resolution:'static'});
|
|
45
|
+
const link=(type,line)=>{if(type&&type.unit.path!==unit.path&&room(dependencies,MAX_EDGES,'edges'))dependencies.push({source:unit.path,target:type.unit.path,specifier:type.fqn,line,kind:'java_type',resolution:'resolved'})};
|
|
46
|
+
if(constructed&&room(calls,MAX_EDGES,'edges'))calls.push({source:(owner||cls.symbol).id,target:constructed.symbol.id,line:n.location.startLine,kind:'construct',resolution:'static'});
|
|
47
47
|
link(constructed,n.location.startLine);
|
|
48
48
|
const staticHead=/^([A-Z][\w$]*)\./.exec(prefixText);if(staticHead&&!vars.has(staticHead[1]))link(resolveType(unit,staticHead[1],cls),n.location.startLine);
|
|
49
49
|
for(const suffix of suffixes){const invoke=suffix.children.methodInvocationSuffix?.[0];if(!invoke){chain+=text(unit,suffix);continue}const args=invoke.children.argumentList?.[0],arity=args?(args.children.expression||[]).length:0;const match=/^(?:([\w.$]+)\.)?([\w$]+)$/.exec(chain);let target=null;
|
|
50
50
|
if(match&&!fluent){const receiver=match[1],name=match[2];let type=null;if(!receiver||receiver==='this')type=cls;else if(receiver==='super')type=resolveType(unit,cls.extends,cls);else if(receiver.startsWith('this.'))type=resolveType(unit,cls.fields.get(receiver.slice(5))||'',cls);else if(vars.has(receiver))type=resolveType(unit,vars.get(receiver)||'',cls);else{type=resolveType(unit,receiver,cls);link(type,invoke.location.startLine)}target=lookupMethod(type,name,arity);if(!target&&!receiver){const candidates=unit.imports.filter(i=>i.static&&(i.spec.endsWith('.'+name)||i.spec.endsWith('.*'))).map(i=>lookupMethod(classes.get(i.spec.split('.').slice(0,-1).join('.')),name,arity)).filter(Boolean);if(candidates.length===1)target=candidates[0]}}
|
|
51
51
|
if(constructed&&!fluent){const methodName=/\.([\w$]+)$/.exec(chain)?.[1];if(methodName)target=lookupMethod(constructed,methodName,arity)}
|
|
52
|
-
if(target&&room(calls,MAX_EDGES))calls.push({source:(owner||cls.symbol).id,target:target.id,line:invoke.location.startLine,kind:'call',resolution:'static'});
|
|
52
|
+
if(target&&room(calls,MAX_EDGES,'edges'))calls.push({source:(owner||cls.symbol).id,target:target.id,line:invoke.location.startLine,kind:'call',resolution:'static'});
|
|
53
53
|
else if(unresolved.length<1000)unresolved.push({path:unit.path,line:invoke.location.startLine,expression:chain.slice(0,140),reason:'External/dynamic type, interface without implementation, or ambiguous overload in the Java snapshot.'});
|
|
54
54
|
fluent=true;chain+='<result>';
|
|
55
55
|
}
|
|
@@ -57,5 +57,6 @@ for(const unit of units){
|
|
|
57
57
|
for(const child of children(n))walk(child,cls,method,vars,owner)
|
|
58
58
|
}walk(unit.tree)
|
|
59
59
|
}
|
|
60
|
-
const ids=new Set(symbols.map(s=>s.id));
|
|
61
|
-
|
|
60
|
+
const ids=new Set(symbols.map(s=>s.id)),linked=calls.filter(c=>ids.has(c.source)&&ids.has(c.target));
|
|
61
|
+
leftOut.edges+=calls.length-linked.length;
|
|
62
|
+
process.stdout.write(JSON.stringify({symbols,dependencies,calls:linked,unresolved,diagnostics,parsed:units.map(u=>u.path),left_out:leftOut}));
|
package/code_graph_js.cjs
CHANGED
|
@@ -66,8 +66,8 @@ const host={
|
|
|
66
66
|
const program=ts.createProgram(names,options,host),checker=program.getTypeChecker();
|
|
67
67
|
const symbols=[],dependencies=[],calls=[],unresolved=[],diagnostics=[],declarations=new Map(),moduleNodes=new Map();
|
|
68
68
|
const {symbols:maxSymbols,edges:maxEdges}=input.limits;
|
|
69
|
-
|
|
70
|
-
function room(list,max){if(list.length<max)return true;
|
|
69
|
+
const leftOut={symbols:0,edges:0};
|
|
70
|
+
function room(list,max,kind){if(list.length<max)return true;leftOut[kind]++;return false;}
|
|
71
71
|
function filePath(source){return source.fileName.replace('/snapshot/','');}
|
|
72
72
|
function line(source,pos){return source.getLineAndCharacterOfPosition(pos).line+1;}
|
|
73
73
|
function displayName(node) {
|
|
@@ -81,7 +81,7 @@ function displayName(node) {
|
|
|
81
81
|
}
|
|
82
82
|
function isFunction(node){return !!node.body && (ts.isFunctionDeclaration(node)||ts.isMethodDeclaration(node)||ts.isFunctionExpression(node)||ts.isArrowFunction(node)||ts.isConstructorDeclaration(node)||ts.isGetAccessor(node)||ts.isSetAccessor(node));}
|
|
83
83
|
function addSymbol(node,source,owner){
|
|
84
|
-
if(!room(symbols,maxSymbols))return null;
|
|
84
|
+
if(!room(symbols,maxSymbols,'symbols'))return null;
|
|
85
85
|
let name=displayName(node),parent=node.parent;
|
|
86
86
|
const classParent=parent&&ts.isPropertyDeclaration(parent)?parent.parent:parent;
|
|
87
87
|
const cls=classParent && (ts.isClassDeclaration(classParent)||ts.isClassExpression(classParent)) ? classParent.name?.getText() : null;
|
|
@@ -116,16 +116,16 @@ for(const source of parsed){
|
|
|
116
116
|
const active=declarations.get(node)||owner;
|
|
117
117
|
if((ts.isImportDeclaration(node)||ts.isExportDeclaration(node)) && node.moduleSpecifier && ts.isStringLiteral(node.moduleSpecifier)) {
|
|
118
118
|
const spec=node.moduleSpecifier.text,target=resolve(spec,source.fileName);
|
|
119
|
-
if(room(dependencies,maxEdges))dependencies.push({source:filePath(source),target:target?.replace('/snapshot/','')||null,specifier:spec,line:line(source,node.getStart(source)),kind:node.importClause?.isTypeOnly||node.isTypeOnly?'type_import':'import',resolution:target?'resolved':spec.startsWith('.')?'unresolved':'external'});
|
|
119
|
+
if(room(dependencies,maxEdges,'edges'))dependencies.push({source:filePath(source),target:target?.replace('/snapshot/','')||null,specifier:spec,line:line(source,node.getStart(source)),kind:node.importClause?.isTypeOnly||node.isTypeOnly?'type_import':'import',resolution:target?'resolved':spec.startsWith('.')?'unresolved':'external'});
|
|
120
120
|
}
|
|
121
121
|
if(ts.isCallExpression(node)||ts.isNewExpression(node)){
|
|
122
122
|
const expression=node.expression,at=line(source,node.getStart(source));
|
|
123
123
|
if(ts.isCallExpression(node) && (expression.kind===ts.SyntaxKind.ImportKeyword || ts.isIdentifier(expression)&&expression.text==='require') && node.arguments?.length && ts.isStringLiteral(node.arguments[0])){
|
|
124
124
|
const spec=node.arguments[0].text,target=resolve(spec,source.fileName);
|
|
125
|
-
if(room(dependencies,maxEdges))dependencies.push({source:filePath(source),target:target?.replace('/snapshot/','')||null,specifier:spec,line:at,kind:'dynamic_import',resolution:target?'resolved':spec.startsWith('.')?'unresolved':'external'});
|
|
125
|
+
if(room(dependencies,maxEdges,'edges'))dependencies.push({source:filePath(source),target:target?.replace('/snapshot/','')||null,specifier:spec,line:at,kind:'dynamic_import',resolution:target?'resolved':spec.startsWith('.')?'unresolved':'external'});
|
|
126
126
|
}else{
|
|
127
127
|
const target=targetFor(node);
|
|
128
|
-
if(target && room(calls,maxEdges))calls.push({source:(active||moduleSymbol(source)).id,target:target.id,line:at,kind:ts.isNewExpression(node)?'construct':'call',resolution:'static'});
|
|
128
|
+
if(target && room(calls,maxEdges,'edges'))calls.push({source:(active||moduleSymbol(source)).id,target:target.id,line:at,kind:ts.isNewExpression(node)?'construct':'call',resolution:'static'});
|
|
129
129
|
else if(unresolved.length<1000)unresolved.push({path:filePath(source),line:at,expression:expression.getText(source).slice(0,120),reason:'External or dynamic target, or no resolved implementation in the snapshot.'});
|
|
130
130
|
}
|
|
131
131
|
}
|
|
@@ -144,8 +144,8 @@ for(const source of parsed){
|
|
|
144
144
|
if(components.some(c=>c.template!=null)){
|
|
145
145
|
try{
|
|
146
146
|
const {parseTemplate}=require('./web_adapters/node/node_modules/@angular/compiler/fesm2022/compiler.mjs');
|
|
147
|
-
for(const component of components){if(component.template==null||!component.templatePath)continue;const pathName=component.templatePath.replace('/snapshot/',''),tree=parseTemplate(component.template,pathName);if(tree.errors?.length){diagnostics.push({path:pathName,reason:'Invalid Angular template or syntax not supported in this version.'});continue}templatePaths.add(pathName);const owner={id:pathName+'::'+filePath(component.source)+'::'+component.name+'.template@'+component.inlineLine,path:pathName,name:component.name+'.template',kind:'template',language:'angular',start_line:component.inlineLine,end_line:component.inlineLine+component.template.split('\n').length-1,location_kind:component.templatePath===component.source.fileName?'inline_template':'source_line'};if(!room(symbols,maxSymbols))break;symbols.push(owner);const shadow=new Set(),seen=new WeakSet();function walk(n,fn){if(!n||typeof n!=='object'||seen.has(n))return;seen.add(n);fn(n);for(const [key,value] of Object.entries(n)){if(['sourceSpan','span','startSourceSpan','endSourceSpan','keySpan','valueSpan','parent'].includes(key))continue;if(Array.isArray(value))value.forEach(x=>walk(x,fn));else if(value&&typeof value==='object')walk(value,fn)}}walk(tree.nodes,n=>{if(['Variable','Reference','LetDeclaration'].includes(n.constructor.name)&&n.name)shadow.add(n.name);if(n.constructor.name==='ForLoopBlock'&&n.item?.name)shadow.add(n.item.name)});const seen2=new WeakSet();function inspect(n){if(!n||typeof n!=='object'||seen2.has(n))return;seen2.add(n);const start=n.sourceSpan?.start,at=(typeof start==='number'?component.template.slice(0,start).split('\n').length-1:(start?.line??0))+component.inlineLine;if(n.constructor.name==='Call'||n.constructor.name==='SafeCall'){const receiver=n.receiver;if(receiver?.constructor.name==='PropertyRead'&&['ImplicitReceiver','ThisReceiver'].includes(receiver.receiver?.constructor.name)){const target=shadow.has(receiver.name)?null:component.methods.get(receiver.name);if(target&&room(calls,maxEdges))calls.push({source:owner.id,target:target.id,line:at,kind:'template_call',resolution:'static'});else if(unresolved.length<1000)unresolved.push({path:pathName,line:at,expression:receiver.name+'()',reason:'Unresolved template method, or a local variable that shadows the component.'})}}if(n.constructor.name==='Element'){const matches=components.filter(c=>c.selector===n.name);if(matches.length===1&&filePath(matches[0].source)!==pathName)dependencies.push({source:pathName,target:filePath(matches[0].source),specifier:n.name,line:at,kind:'angular_component',resolution:'resolved'})}if(n.constructor.name==='BindingPipe'){const matches=pipes.filter(p=>p.pipeName===n.name&&p.methods.has('transform'));if(matches.length===1){const pipe=matches[0];dependencies.push({source:pathName,target:filePath(pipe.source),specifier:n.name,line:at,kind:'angular_pipe',resolution:'resolved'});calls.push({source:owner.id,target:pipe.methods.get('transform').id,line:at,kind:'template_pipe',resolution:'static'})}}for(const [key,value] of Object.entries(n)){if(['sourceSpan','span','startSourceSpan','endSourceSpan','keySpan','valueSpan','parent'].includes(key))continue;if(Array.isArray(value))value.forEach(inspect);else if(value&&typeof value==='object')inspect(value)}}inspect(tree.nodes)
|
|
147
|
+
for(const component of components){if(component.template==null||!component.templatePath)continue;const pathName=component.templatePath.replace('/snapshot/',''),tree=parseTemplate(component.template,pathName);if(tree.errors?.length){diagnostics.push({path:pathName,reason:'Invalid Angular template or syntax not supported in this version.'});continue}templatePaths.add(pathName);const owner={id:pathName+'::'+filePath(component.source)+'::'+component.name+'.template@'+component.inlineLine,path:pathName,name:component.name+'.template',kind:'template',language:'angular',start_line:component.inlineLine,end_line:component.inlineLine+component.template.split('\n').length-1,location_kind:component.templatePath===component.source.fileName?'inline_template':'source_line'};if(!room(symbols,maxSymbols,'symbols'))break;symbols.push(owner);const shadow=new Set(),seen=new WeakSet();function walk(n,fn){if(!n||typeof n!=='object'||seen.has(n))return;seen.add(n);fn(n);for(const [key,value] of Object.entries(n)){if(['sourceSpan','span','startSourceSpan','endSourceSpan','keySpan','valueSpan','parent'].includes(key))continue;if(Array.isArray(value))value.forEach(x=>walk(x,fn));else if(value&&typeof value==='object')walk(value,fn)}}walk(tree.nodes,n=>{if(['Variable','Reference','LetDeclaration'].includes(n.constructor.name)&&n.name)shadow.add(n.name);if(n.constructor.name==='ForLoopBlock'&&n.item?.name)shadow.add(n.item.name)});const seen2=new WeakSet();function inspect(n){if(!n||typeof n!=='object'||seen2.has(n))return;seen2.add(n);const start=n.sourceSpan?.start,at=(typeof start==='number'?component.template.slice(0,start).split('\n').length-1:(start?.line??0))+component.inlineLine;if(n.constructor.name==='Call'||n.constructor.name==='SafeCall'){const receiver=n.receiver;if(receiver?.constructor.name==='PropertyRead'&&['ImplicitReceiver','ThisReceiver'].includes(receiver.receiver?.constructor.name)){const target=shadow.has(receiver.name)?null:component.methods.get(receiver.name);if(target&&room(calls,maxEdges,'edges'))calls.push({source:owner.id,target:target.id,line:at,kind:'template_call',resolution:'static'});else if(unresolved.length<1000)unresolved.push({path:pathName,line:at,expression:receiver.name+'()',reason:'Unresolved template method, or a local variable that shadows the component.'})}}if(n.constructor.name==='Element'){const matches=components.filter(c=>c.selector===n.name);if(matches.length===1&&filePath(matches[0].source)!==pathName)dependencies.push({source:pathName,target:filePath(matches[0].source),specifier:n.name,line:at,kind:'angular_component',resolution:'resolved'})}if(n.constructor.name==='BindingPipe'){const matches=pipes.filter(p=>p.pipeName===n.name&&p.methods.has('transform'));if(matches.length===1){const pipe=matches[0];dependencies.push({source:pathName,target:filePath(pipe.source),specifier:n.name,line:at,kind:'angular_pipe',resolution:'resolved'});calls.push({source:owner.id,target:pipe.methods.get('transform').id,line:at,kind:'template_pipe',resolution:'static'})}}for(const [key,value] of Object.entries(n)){if(['sourceSpan','span','startSourceSpan','endSourceSpan','keySpan','valueSpan','parent'].includes(key))continue;if(Array.isArray(value))value.forEach(inspect);else if(value&&typeof value==='object')inspect(value)}}inspect(tree.nodes)
|
|
148
148
|
}
|
|
149
149
|
}catch{diagnostics.push({path:'Angular',reason:'The Angular template analyzer did not finish. Check the local runtime.'})}
|
|
150
150
|
}
|
|
151
|
-
process.stdout.write(JSON.stringify({symbols,dependencies
|
|
151
|
+
process.stdout.write(JSON.stringify({symbols,dependencies,calls,unresolved,diagnostics,parsed:[...parsed.map(filePath),...templatePaths],left_out:leftOut}));
|
package/code_health.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Questions about the shape of a project that the code graph and the git history answer without writing documentation:
|
|
2
|
+
which code files are the riskiest to change, which functions nothing calls, which imports form a cycle and which
|
|
3
|
+
comments sit where they should not.
|
|
4
|
+
|
|
5
|
+
The same tables the `docs/scope` pages hold (`doc_insights`), asked directly by `project_manage`; nothing is written.
|
|
6
|
+
"""
|
|
7
|
+
import code_graph
|
|
8
|
+
import comment_check
|
|
9
|
+
import doc_check
|
|
10
|
+
import doc_insights
|
|
11
|
+
import doc_scope
|
|
12
|
+
|
|
13
|
+
DEFAULT_LIMIT = 25
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _model(root, view_id):
|
|
17
|
+
model = doc_scope._Model(root, view_id)
|
|
18
|
+
if not model.graph.get("symbols"):
|
|
19
|
+
raise ValueError(f"The code graph of {root} is empty: index the project first (project_manage action=index).")
|
|
20
|
+
return model
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def hotspots(root, view_id=None, limit=DEFAULT_LIMIT):
|
|
24
|
+
"""Code files ranked by the commits that touched them times their lines.
|
|
25
|
+
|
|
26
|
+
Parameters: root (str) project folder; view_id (str or None) index view; limit (int) rows returned.
|
|
27
|
+
Returns: dict {"files": [{"path", "commits", "lines", "score", "last_change"}], "total": int, "commits_read": int,
|
|
28
|
+
"note": str}; `note` is the reason no history could be read (not a git repository), otherwise "".
|
|
29
|
+
Raises: ValueError when the code graph is empty.
|
|
30
|
+
"""
|
|
31
|
+
model = _model(root, view_id)
|
|
32
|
+
commits, note = doc_insights.history(root)
|
|
33
|
+
rows = [] if note else doc_insights.hotspots(model, commits)
|
|
34
|
+
files = [{"path": path, "commits": count, "lines": lines, "score": count * lines, "last_change": doc_insights._date(last)}
|
|
35
|
+
for path, count, lines, last in rows[:limit]]
|
|
36
|
+
return {"files": files, "total": len(rows), "commits_read": len(commits), "note": note or ""}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def dead_code(root, view_id=None, limit=DEFAULT_LIMIT):
|
|
40
|
+
"""Functions nothing calls and functions only tests call, from the static call graph.
|
|
41
|
+
|
|
42
|
+
Parameters: root (str) project folder; view_id (str or None) index view; limit (int) entries per list.
|
|
43
|
+
Returns: dict {"no_caller": [...], "only_tests": [...], "counts": {"no_caller", "only_tests"}, "note": str}; each entry
|
|
44
|
+
is {"name", "path", "line", "public"}. Candidates only: a call through a framework, a decorator registration,
|
|
45
|
+
reflection or a name built at run time is not seen.
|
|
46
|
+
Raises: ValueError when the code graph is empty.
|
|
47
|
+
"""
|
|
48
|
+
model = _model(root, view_id)
|
|
49
|
+
alone, tested = doc_insights.dead_code(model, model.entry_points())
|
|
50
|
+
|
|
51
|
+
def entry(anchor):
|
|
52
|
+
function = model.function_of[anchor]
|
|
53
|
+
return {"name": function["name"], "path": function["path"], "line": function["start"],
|
|
54
|
+
"public": bool(function["required"])}
|
|
55
|
+
|
|
56
|
+
return {"no_caller": [entry(a) for a in alone[:limit]], "only_tests": [entry(a) for a in tested[:limit]],
|
|
57
|
+
"counts": {"no_caller": len(alone), "only_tests": len(tested)},
|
|
58
|
+
"note": "Candidates from the static call graph (Python and Go only): read each one before deleting it."}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def coupled_with(root, paths, view_id=None, limit=DEFAULT_LIMIT):
|
|
62
|
+
"""Files that usually change in the same commits as the given ones, left out of the given set.
|
|
63
|
+
|
|
64
|
+
Parameters: root (str) project folder; paths (list of str) project-relative changed files; view_id (str or None) index
|
|
65
|
+
view; limit (int) rows returned. Pairs where one file imports the other are skipped: the call graph already shows them.
|
|
66
|
+
Returns: dict {"files": [{"path", "with", "shared", "degree"}], "note": str}, strongest first, one row per partner;
|
|
67
|
+
degree is a percentage; `note` is the reason no history could be read, otherwise "".
|
|
68
|
+
Raises: ValueError when the code graph is empty.
|
|
69
|
+
"""
|
|
70
|
+
model = _model(root, view_id)
|
|
71
|
+
commits, note = doc_insights.history(root)
|
|
72
|
+
if note:
|
|
73
|
+
return {"files": [], "note": note}
|
|
74
|
+
changed, rows, seen = set(paths), [], set()
|
|
75
|
+
for first, second, shared, degree, linked in doc_insights.coupling(model, commits):
|
|
76
|
+
if linked or (first in changed) == (second in changed):
|
|
77
|
+
continue
|
|
78
|
+
partner, anchor = (second, first) if first in changed else (first, second)
|
|
79
|
+
if partner not in seen:
|
|
80
|
+
seen.add(partner)
|
|
81
|
+
rows.append({"path": partner, "with": anchor, "shared": shared, "degree": int(degree * 100)})
|
|
82
|
+
return {"files": rows[:limit], "note": ""}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _pop_component(stack, on_stack, node):
|
|
86
|
+
"""Members of the component rooted at `node`, taken off the Tarjan stack."""
|
|
87
|
+
group = []
|
|
88
|
+
member = None
|
|
89
|
+
while member != node:
|
|
90
|
+
member = stack.pop()
|
|
91
|
+
on_stack.discard(member)
|
|
92
|
+
group.append(member)
|
|
93
|
+
return sorted(group)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _components(edges):
|
|
97
|
+
"""Strongly connected components of size above one, and self-loops, of a directed graph (Tarjan, no recursion).
|
|
98
|
+
|
|
99
|
+
Parameters: edges (dict) node -> set of the nodes it points to.
|
|
100
|
+
Returns: list of sorted node lists.
|
|
101
|
+
"""
|
|
102
|
+
index, low, stack, on_stack, found = {}, {}, [], set(), []
|
|
103
|
+
for start in sorted(edges):
|
|
104
|
+
if start in index:
|
|
105
|
+
continue
|
|
106
|
+
index[start] = low[start] = len(index)
|
|
107
|
+
stack.append(start)
|
|
108
|
+
on_stack.add(start)
|
|
109
|
+
work = [(start, iter(sorted(edges[start])))]
|
|
110
|
+
while work:
|
|
111
|
+
node, children = work[-1]
|
|
112
|
+
descended = False
|
|
113
|
+
for child in children:
|
|
114
|
+
if child not in index:
|
|
115
|
+
index[child] = low[child] = len(index)
|
|
116
|
+
stack.append(child)
|
|
117
|
+
on_stack.add(child)
|
|
118
|
+
work.append((child, iter(sorted(edges[child]))))
|
|
119
|
+
descended = True
|
|
120
|
+
break
|
|
121
|
+
if child in on_stack:
|
|
122
|
+
low[node] = min(low[node], index[child])
|
|
123
|
+
if descended:
|
|
124
|
+
continue
|
|
125
|
+
work.pop()
|
|
126
|
+
if work:
|
|
127
|
+
parent = work[-1][0]
|
|
128
|
+
low[parent] = min(low[parent], low[node])
|
|
129
|
+
if low[node] == index[node]:
|
|
130
|
+
group = _pop_component(stack, on_stack, node)
|
|
131
|
+
if len(group) > 1 or node in edges[node]:
|
|
132
|
+
found.append(group)
|
|
133
|
+
return found
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def import_cycles(root, view_id=None, limit=DEFAULT_LIMIT):
|
|
137
|
+
"""Groups of files that import each other, directly or through a chain.
|
|
138
|
+
|
|
139
|
+
Parameters: root (str) project folder; view_id (str or None) index view; limit (int) cycles returned.
|
|
140
|
+
Returns: dict {"cycles": [{"files": [...], "size": int}], "total": int, "files_in_cycles": int, "note": str}, the
|
|
141
|
+
largest cycle first; only imports the graph resolved to a file of the project count.
|
|
142
|
+
Raises: ValueError when the code graph is empty.
|
|
143
|
+
"""
|
|
144
|
+
data = code_graph.build_current(root, view_id, shared=True)
|
|
145
|
+
if not data.get("symbols") and not data.get("dependencies"):
|
|
146
|
+
raise ValueError(f"The code graph of {root} is empty: index the project first (project_manage action=index).")
|
|
147
|
+
edges = {}
|
|
148
|
+
for dependency in data.get("dependencies") or []:
|
|
149
|
+
if dependency.get("kind") == "import" and dependency.get("resolution") == "resolved" and dependency.get("target"):
|
|
150
|
+
edges.setdefault(dependency["source"], set()).add(dependency["target"])
|
|
151
|
+
edges.setdefault(dependency["target"], set())
|
|
152
|
+
cycles = sorted(_components(edges), key=lambda files: (-len(files), files))
|
|
153
|
+
note = code_graph.left_out_sentence(data) if data.get("truncated") else ""
|
|
154
|
+
return {"cycles": [{"files": files, "size": len(files)} for files in cycles[:limit]], "total": len(cycles),
|
|
155
|
+
"files_in_cycles": sum(len(files) for files in cycles), "note": note}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def comments(root, view_id=None, limit=DEFAULT_LIMIT):
|
|
159
|
+
"""Comments in the wrong place across the code files of the project: commented-out code, history tags, loose
|
|
160
|
+
documentation and long inline blocks (`comment_check`), for files written before the edit hook existed.
|
|
161
|
+
|
|
162
|
+
Parameters: root (str) project folder; view_id (str or None) index view; limit (int) findings returned.
|
|
163
|
+
Returns: dict {"findings": [{"path", "line", "issue", "name", "text"}], "total": int, "by_issue": {issue: count},
|
|
164
|
+
"files_checked": int}.
|
|
165
|
+
Raises: ValueError when the code graph is empty.
|
|
166
|
+
"""
|
|
167
|
+
model = _model(root, view_id)
|
|
168
|
+
findings, checked = [], 0
|
|
169
|
+
for path in model.files():
|
|
170
|
+
if model.kind_of(path) != "code" or not doc_check.supported(path):
|
|
171
|
+
continue
|
|
172
|
+
text = model.read(path)
|
|
173
|
+
if not text:
|
|
174
|
+
continue
|
|
175
|
+
checked += 1
|
|
176
|
+
findings += [{"path": path, **item} for item in comment_check.review(path, None, text)]
|
|
177
|
+
by_issue = {}
|
|
178
|
+
for item in findings:
|
|
179
|
+
by_issue[item["issue"]] = by_issue.get(item["issue"], 0) + 1
|
|
180
|
+
return {"findings": findings[:limit], "total": len(findings), "by_issue": by_issue, "files_checked": checked}
|
package/code_impact.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""What changing a function touches, from the indexed code graph: where it is defined, who calls it, what it calls,
|
|
2
2
|
which tests reach it through static calls and which files import its module, each with the first line of its
|
|
3
3
|
docstring. Answers the question an agent asks before an edit without a chain of searches and file reads."""
|
|
4
|
+
import difflib
|
|
4
5
|
import os
|
|
5
6
|
import re
|
|
6
7
|
from collections import defaultdict
|
|
@@ -23,6 +24,15 @@ def _matches(symbols, name):
|
|
|
23
24
|
return exact or [s for s in pool if s["name"].split(".")[-1] == wanted]
|
|
24
25
|
|
|
25
26
|
|
|
27
|
+
def _closest_names(symbols, name, limit=3):
|
|
28
|
+
"""Up to `limit` indexed function or class names that resemble `name` (case ignored), best first."""
|
|
29
|
+
wanted = name.strip().rpartition("::")[2].casefold()
|
|
30
|
+
by_lower = {}
|
|
31
|
+
for symbol in symbols:
|
|
32
|
+
by_lower.setdefault(symbol["name"].split(".")[-1].casefold(), symbol["name"].split(".")[-1])
|
|
33
|
+
return [by_lower[key] for key in difflib.get_close_matches(wanted, list(by_lower), n=limit, cutoff=0.6)]
|
|
34
|
+
|
|
35
|
+
|
|
26
36
|
def _site(by_id, symbol_id, line, of=None):
|
|
27
37
|
symbol = by_id.get(symbol_id) or {}
|
|
28
38
|
site = {"id": symbol_id, "function": symbol.get("name", symbol_id), "path": symbol.get("path"), "line": line}
|
|
@@ -85,7 +95,9 @@ def impact(root, symbol, view_id=None, depth=3, limit=30):
|
|
|
85
95
|
diagnostics = [d.get("reason") for d in data.get("diagnostics") or [] if d.get("reason")]
|
|
86
96
|
if not found:
|
|
87
97
|
cause = f" Analysis problems: {'; '.join(diagnostics)}" if diagnostics else ""
|
|
88
|
-
|
|
98
|
+
close = _closest_names(data.get("symbols") or [], symbol)
|
|
99
|
+
hint = f" Closest names: {', '.join(close)}." if close else ""
|
|
100
|
+
raise ValueError(f"No function or class named {symbol!r} in the indexed view; check the name or reindex.{hint}{cause}")
|
|
89
101
|
profile = index_profile.current((index_scope.load_scope(root) or {}).get("profile"))
|
|
90
102
|
by_id = {s["id"]: s for s in data["symbols"]}
|
|
91
103
|
callers, callees = defaultdict(list), defaultdict(list)
|
|
@@ -143,7 +155,9 @@ def impact(root, symbol, view_id=None, depth=3, limit=30):
|
|
|
143
155
|
"diagnostics": diagnostics,
|
|
144
156
|
"view": (data.get("selected") or {}).get("label"),
|
|
145
157
|
"note": NOTE + (f" {len(found)} definitions share this name; each entry says which one ('of'); pass "
|
|
146
|
-
"path::name to keep one." if label else "")
|
|
158
|
+
"path::name to keep one." if label else "")
|
|
159
|
+
+ (f" {code_graph.left_out_sentence(data)} Callers and tests beyond the cut are not listed." if data["truncated"] else ""),
|
|
160
|
+
**({"left_out": data["left_out"]} if data["truncated"] else {}),
|
|
147
161
|
}
|
|
148
162
|
|
|
149
163
|
|
package/config.py
CHANGED
|
@@ -40,12 +40,15 @@ DEFAULT_CONFIG = {
|
|
|
40
40
|
"research_max_iterations": 3,
|
|
41
41
|
"quick_search_timeout_s": 120,
|
|
42
42
|
"site_memory_similarity_threshold": 0.35,
|
|
43
|
-
"
|
|
43
|
+
"embedder": "home",
|
|
44
44
|
"hook_mode": "redirect",
|
|
45
45
|
"doc_mode": "require",
|
|
46
46
|
"comment_mode": "require",
|
|
47
47
|
"duplicate_mode": "warn",
|
|
48
48
|
"dependency_mode": "warn",
|
|
49
|
+
"layer_mode": "warn",
|
|
50
|
+
"document_converter": "off",
|
|
51
|
+
"document_converter_python": "",
|
|
49
52
|
"docs_extras": {key: True for key in DOCS_EXTRAS},
|
|
50
53
|
"auto_update": True,
|
|
51
54
|
"language": "",
|
|
@@ -60,9 +63,10 @@ DEFAULT_CONFIG = {
|
|
|
60
63
|
},
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
|
|
66
|
+
# Embedder of the project indexes: home = the embeddings home daemon of this machine (casa-de-embeddings), whose rerank
|
|
67
|
+
# connector forwards to the host configured in it; gateway = the gateway's embedding_model and rerank_model, only when
|
|
68
|
+
# the user chooses it. Neither replaces the other when it fails.
|
|
69
|
+
EMBEDDERS = ("home", "gateway")
|
|
66
70
|
# PreToolUse hook: redirect = deny the native tool and point to Smart Tool; advise = let it run and add the tip to the
|
|
67
71
|
# agent's context; off = no routing at all.
|
|
68
72
|
HOOK_MODES = ("redirect", "advise", "off")
|
|
@@ -79,6 +83,11 @@ DUPLICATE_MODES = ("warn", "off")
|
|
|
79
83
|
# Edit hook on package.json, requirements*.txt and go.mod: warn = let the edit run and list the added or changed dependencies that have OSV advisories;
|
|
80
84
|
# off = say nothing and make no network call.
|
|
81
85
|
DEPENDENCY_MODES = ("warn", "off")
|
|
86
|
+
LAYER_MODES = ("warn", "off")
|
|
87
|
+
# External converter read_document falls back to for a file type or a scanned PDF its native reader refuses: off = never;
|
|
88
|
+
# auto = the first of kreuzberg and docling found installed; or one of them by name. The engine is looked for in the
|
|
89
|
+
# interpreter named by `document_converter_python` (empty = the one Smart Tool runs on).
|
|
90
|
+
DOCUMENT_CONVERTERS = ("off", "auto", "kreuzberg", "docling")
|
|
82
91
|
# Router model reviewing, in the background, searches the deterministic redirect rule let run: on = propose block
|
|
83
92
|
# patterns for the user to accept in the setup screen; off = no review.
|
|
84
93
|
PATTERN_PROPOSAL_MODES = ("on", "off")
|
|
@@ -118,6 +127,26 @@ def dependency_mode(cfg=None):
|
|
|
118
127
|
return value
|
|
119
128
|
|
|
120
129
|
|
|
130
|
+
def layer_mode(cfg=None):
|
|
131
|
+
value = (cfg if cfg is not None else load_config()).get("layer_mode") or DEFAULT_CONFIG["layer_mode"]
|
|
132
|
+
if value not in LAYER_MODES:
|
|
133
|
+
raise ValueError(f"Invalid layer_mode ({value!r}) in {CONFIG_PATH}: use warn or off.")
|
|
134
|
+
return value
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def document_converter(cfg=None):
|
|
138
|
+
"""The external converter setting: `{"mode": one of DOCUMENT_CONVERTERS, "python": interpreter path or ""}`. An
|
|
139
|
+
unknown mode or a python that is not text fails loudly."""
|
|
140
|
+
stored = cfg if cfg is not None else load_config()
|
|
141
|
+
mode = stored.get("document_converter") or DEFAULT_CONFIG["document_converter"]
|
|
142
|
+
python = stored.get("document_converter_python") or ""
|
|
143
|
+
if mode not in DOCUMENT_CONVERTERS:
|
|
144
|
+
raise ValueError(f"Invalid document_converter ({mode!r}) in {CONFIG_PATH}: use off, auto, kreuzberg or docling.")
|
|
145
|
+
if not isinstance(python, str):
|
|
146
|
+
raise ValueError(f"Invalid document_converter_python in {CONFIG_PATH}: a path to a Python interpreter.")
|
|
147
|
+
return {"mode": mode, "python": python}
|
|
148
|
+
|
|
149
|
+
|
|
121
150
|
def docs_extras(cfg=None):
|
|
122
151
|
"""Which extra documents the docs generator writes: every key of DOCS_EXTRAS, true unless the stored `docs_extras`
|
|
123
152
|
object sets it to false; an unknown key or a value that is not true or false fails loudly."""
|
|
@@ -183,10 +212,14 @@ def hook_mode(cfg=None):
|
|
|
183
212
|
return value
|
|
184
213
|
|
|
185
214
|
|
|
186
|
-
def
|
|
187
|
-
|
|
188
|
-
if
|
|
189
|
-
raise ValueError(f"
|
|
215
|
+
def embedder(cfg=None):
|
|
216
|
+
cfg = cfg if cfg is not None else load_config()
|
|
217
|
+
if "local_models" in cfg:
|
|
218
|
+
raise ValueError(f"{CONFIG_PATH} has the old 'local_models' setting: delete it and choose the embedder (home or "
|
|
219
|
+
"gateway) in the setup screen.")
|
|
220
|
+
value = cfg.get("embedder") or DEFAULT_CONFIG["embedder"]
|
|
221
|
+
if value not in EMBEDDERS:
|
|
222
|
+
raise ValueError(f"Invalid embedder ({value!r}) in {CONFIG_PATH}: use home or gateway.")
|
|
190
223
|
return value
|
|
191
224
|
|
|
192
225
|
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Credentials an edit writes, for the edit hook. Only the lines an edit adds or changes are read, and only for shapes that
|
|
2
|
+
cannot be anything else (a provider's token prefix with its fixed length, an AWS key pair line, a private-key header at
|
|
3
|
+
the start of a line): a false alarm costs more than a silence here, so a general "high entropy string" rule is not used.
|
|
4
|
+
Lines that say they are an example are skipped. The matched text is never repeated in a note."""
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
import doc_check
|
|
8
|
+
|
|
9
|
+
_PLACEHOLDER = re.compile(r"example|placeholder|your[_-]|dummy|fake|redacted|x{6,}|\*{4,}|<[^>]*>", re.I)
|
|
10
|
+
_PATTERNS = (
|
|
11
|
+
("private key", re.compile(r"^\s*-----BEGIN (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----")),
|
|
12
|
+
("AWS access key id", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
|
|
13
|
+
("AWS secret access key", re.compile(r"aws_secret_access_key[\"']?\s*[=:]\s*[\"']?[A-Za-z0-9/+=]{40}\b", re.I)),
|
|
14
|
+
("GitHub token", re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{50,})")),
|
|
15
|
+
("Slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
|
|
16
|
+
("Google API key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")),
|
|
17
|
+
("Stripe live key", re.compile(r"\b[sr]k_live_[0-9A-Za-z]{24,}")),
|
|
18
|
+
("Anthropic API key", re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}")),
|
|
19
|
+
("npm token", re.compile(r"\bnpm_[A-Za-z0-9]{36}\b")),
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def review(before, after):
|
|
24
|
+
"""Credentials among the lines an edit adds or changes.
|
|
25
|
+
|
|
26
|
+
Parameters: before (str or None) text before the edit; after (str) text after it.
|
|
27
|
+
Returns: list of dicts with line (1-based) and kind, in line order, one per line; the secret itself is not included.
|
|
28
|
+
"""
|
|
29
|
+
lines = after.split("\n")
|
|
30
|
+
findings = []
|
|
31
|
+
for number in sorted(doc_check._touched_lines(before, after)):
|
|
32
|
+
if number > len(lines) or _PLACEHOLDER.search(lines[number - 1]):
|
|
33
|
+
continue
|
|
34
|
+
for kind, pattern in _PATTERNS:
|
|
35
|
+
if pattern.search(lines[number - 1]):
|
|
36
|
+
findings.append({"line": number, "kind": kind})
|
|
37
|
+
break
|
|
38
|
+
return findings
|