@aqaf/cli 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -3
- package/index.js +20 -1
- package/package.json +4 -3
- package/vendor/aqaf/__init__.py +1 -1
- package/vendor/aqaf/adapters.py +583 -0
- package/vendor/aqaf/artifacts.py +28 -0
- package/vendor/aqaf/calibration.py +29 -0
- package/vendor/aqaf/catalogue.py +257 -0
- package/vendor/aqaf/cli.py +171 -33
- package/vendor/aqaf/cursor.py +56 -0
- package/vendor/aqaf/data/book_review_knowledge.json +1574 -0
- package/vendor/aqaf/data/documentation_checks.json +52 -0
- package/vendor/aqaf/data/review_checks.json +319 -0
- package/vendor/aqaf/data/source_registry.json +513 -1
- package/vendor/aqaf/discovery.py +580 -0
- package/vendor/aqaf/documentation.py +210 -0
- package/vendor/aqaf/evidence.py +132 -0
- package/vendor/aqaf/facets.py +63 -0
- package/vendor/aqaf/ignore.py +109 -0
- package/vendor/aqaf/knowledge.py +110 -0
- package/vendor/aqaf/markup.py +96 -0
- package/vendor/aqaf/matching.py +22 -0
- package/vendor/aqaf/portfolio.py +145 -0
- package/vendor/aqaf/project_commands.py +132 -0
- package/vendor/aqaf/projects.py +157 -0
- package/vendor/aqaf/registry.py +4 -3
- package/vendor/aqaf/review.py +446 -0
- package/vendor/aqaf/scope.py +401 -0
- package/vendor/aqaf/swift_review.py +99 -0
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
"""One read-only inventory, ownership policy and evidence-driven source plan.
|
|
2
|
+
|
|
3
|
+
Target metadata is data. No ignore/config/manifest can expand the authorized root,
|
|
4
|
+
run a command, or cause symlink traversal. Pruned subtrees are not falsely counted
|
|
5
|
+
as fully inventoried; --inventory-all requests a full metadata-only census.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
from collections import Counter,OrderedDict,defaultdict
|
|
9
|
+
import fnmatch
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path,PurePosixPath
|
|
14
|
+
import re
|
|
15
|
+
import stat
|
|
16
|
+
from .ignore import parse_ignore,ignore_decision,ignore_evidence,MAX_IGNORE_BYTES,MAX_RULES
|
|
17
|
+
from .projects import node_for,graph,archived,marker_family,safe_relative,MAX_PROJECTS
|
|
18
|
+
|
|
19
|
+
LEGACY_SKIP={'.git','.hg','.svn','node_modules','.venv','venv','__pycache__','dist','build','.next','.gradle','target','vendor'}
|
|
20
|
+
HARD_SKIP={'.git','.hg','.svn'}
|
|
21
|
+
DEPENDENCIES={'node_modules','vendor','.pnpm-store','.npm','.nuget','.rustup','.tox','.nox','.eggs','site-packages','.intellijPlatform','.rustup'}
|
|
22
|
+
GENERATED={'__pycache__','dist','build','.next','.gradle','target','.nuxt','.svelte-kit','.kotlin','.cxx','TestResults','.build','.swiftpm','CMakeFiles','.vs'}
|
|
23
|
+
METADATA={'.DS_Store','Thumbs.db','desktop.ini'}
|
|
24
|
+
LANGUAGES={'.js':'javascript','.mjs':'javascript','.cjs':'javascript','.ts':'typescript','.mts':'typescript','.cts':'typescript',
|
|
25
|
+
'.jsx':'jsx','.tsx':'tsx','.gs':'javascript','.py':'python','.kt':'kotlin','.kts':'kotlin','.java':'java','.cs':'csharp',
|
|
26
|
+
'.php':'php','.rs':'rust','.ps1':'powershell','.psm1':'powershell','.sh':'shell','.bash':'shell','.zsh':'shell',
|
|
27
|
+
'.sql':'sql','.html':'html','.htm':'html','.swift':'swift','.go':'go','.c':'c','.cpp':'cpp','.cc':'cpp','.h':'c_header',
|
|
28
|
+
'.m':'objective_c','.mm':'objective_cpp','.rb':'ruby','.pl':'perl','.pm':'perl','.groovy':'groovy','.r':'r',
|
|
29
|
+
'.bat':'cmd','.cmd':'cmd','.command':'shell'}
|
|
30
|
+
ROLE_ORDER={'production':0,'source':1,'unknown':2,'tooling':3,'tests':4,'docs':5,'fixture':6,'archive':7,'generated':8,'dependency':9,'metadata':10}
|
|
31
|
+
MAX_CONFIG_BYTES=131072
|
|
32
|
+
MAX_FILES=500000
|
|
33
|
+
MAX_MANIFESTS=2048
|
|
34
|
+
CACHE_BYTES=64*1024*1024
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def matches(relative,patterns):
|
|
38
|
+
return any(fnmatch.fnmatchcase(relative,p) or (p.endswith('/**') and relative==p[:-3]) for p in patterns)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ScopeConfig(dict):
|
|
42
|
+
"""Effective JSON plus explicitly supplied control-file provenance."""
|
|
43
|
+
control=None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def config_document(path):
|
|
47
|
+
if path is None:return {}
|
|
48
|
+
path=Path(path)
|
|
49
|
+
if path.is_symlink() or not path.is_file() or path.stat().st_size>MAX_CONFIG_BYTES:raise ValueError('Scope configuration must be a bounded regular JSON file')
|
|
50
|
+
try:data=json.loads(path.read_text(encoding='utf-8'))
|
|
51
|
+
except (OSError,ValueError,RecursionError) as exc:raise ValueError('Invalid scope configuration') from exc
|
|
52
|
+
document=ScopeConfig(validate_config(data));document.control={'path':str(path.absolute()),'sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'kind':'operator_scope_config'}
|
|
53
|
+
return document
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def validate_config(data):
|
|
57
|
+
if not isinstance(data,dict):raise ValueError('Scope configuration must be an object')
|
|
58
|
+
if set(data)-{'schema_version','include','exclude','source_roots','projects','roles','review_roles','sql_dialect'}:raise ValueError('Unknown scope configuration field')
|
|
59
|
+
if data.get('schema_version',1)!=1:raise ValueError('Unsupported scope configuration version')
|
|
60
|
+
for key in ('include','exclude','source_roots'):
|
|
61
|
+
values=data.get(key,[])
|
|
62
|
+
if not isinstance(values,list) or len(values)>256 or not all(safe_relative(p) for p in values):raise ValueError('Scope '+key+' must be bounded root-relative paths/patterns')
|
|
63
|
+
roles=data.get('roles',{})
|
|
64
|
+
if not isinstance(roles,dict) or len(roles)>256 or not all(safe_relative(p) and r in ROLE_ORDER for p,r in roles.items()):raise ValueError('Invalid scope roles')
|
|
65
|
+
projects=data.get('projects',[])
|
|
66
|
+
if not isinstance(projects,list) or len(projects)>128:raise ValueError('Invalid scope projects')
|
|
67
|
+
for item in projects:
|
|
68
|
+
if not isinstance(item,dict) or set(item)-{'root','profile','source_roots'} or not safe_relative(item.get('root')):raise ValueError('Invalid declared project')
|
|
69
|
+
if item.get('profile') is not None and not re.fullmatch('[a-z][a-z0-9_]{1,63}',str(item['profile'])):raise ValueError('Invalid declared project profile')
|
|
70
|
+
values=item.get('source_roots',[])
|
|
71
|
+
if not isinstance(values,list) or not all(safe_relative(p) for p in values):raise ValueError('Invalid project source roots')
|
|
72
|
+
rr=data.get('review_roles',['production','source','unknown','tooling'])
|
|
73
|
+
if not isinstance(rr,list) or not all(r in ROLE_ORDER for r in rr):raise ValueError('Invalid review roles')
|
|
74
|
+
if data.get('sql_dialect','unknown') not in {'unknown','sqlite','postgresql','mysql','sqlserver'}:raise ValueError('Unsupported SQL dialect')
|
|
75
|
+
return data
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class ReadCache:
|
|
79
|
+
def __init__(self,root,max_bytes=None):
|
|
80
|
+
self.root=root;self.max_bytes=max_bytes;self.enabled=True;self.entries=OrderedDict();self.size=0;self.hits=0;self.reads=0;self.bytes_read=0;self.failures=Counter();self.dir_fds=OrderedDict();self.evidence={}
|
|
81
|
+
|
|
82
|
+
def close(self):
|
|
83
|
+
for fd in self.dir_fds.values():
|
|
84
|
+
try:os.close(fd)
|
|
85
|
+
except OSError:pass
|
|
86
|
+
self.dir_fds.clear()
|
|
87
|
+
|
|
88
|
+
def __del__(self):self.close()
|
|
89
|
+
|
|
90
|
+
def _open(self,relative):
|
|
91
|
+
if os.open in os.supports_dir_fd and hasattr(os,'O_NOFOLLOW'):
|
|
92
|
+
parent=PurePosixPath(relative).parent.as_posix()
|
|
93
|
+
if parent not in self.dir_fds:
|
|
94
|
+
fd=os.open(self.root,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
|
95
|
+
try:
|
|
96
|
+
for part in PurePosixPath(parent).parts:
|
|
97
|
+
child=os.open(part,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=fd);os.close(fd);fd=child
|
|
98
|
+
except BaseException:os.close(fd);raise
|
|
99
|
+
self.dir_fds[parent]=fd
|
|
100
|
+
while len(self.dir_fds)>64:
|
|
101
|
+
_,old=self.dir_fds.popitem(last=False);os.close(old)
|
|
102
|
+
self.dir_fds.move_to_end(parent)
|
|
103
|
+
fd=os.open(PurePosixPath(relative).name,os.O_RDONLY|os.O_NOFOLLOW|os.O_NONBLOCK,dir_fd=self.dir_fds[parent])
|
|
104
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):os.close(fd);raise OSError('Not a regular source file')
|
|
105
|
+
return os.fdopen(fd,'rb')
|
|
106
|
+
candidate=self.root
|
|
107
|
+
for part in PurePosixPath(relative).parts:
|
|
108
|
+
candidate=candidate/part
|
|
109
|
+
if candidate.is_symlink():raise OSError('Source symlink rejected')
|
|
110
|
+
return candidate.open('rb')
|
|
111
|
+
|
|
112
|
+
def read(self,path,limit=1500001):
|
|
113
|
+
path=Path(path)
|
|
114
|
+
try:relative=path.relative_to(self.root)
|
|
115
|
+
except ValueError:return b''
|
|
116
|
+
key=relative.as_posix();cached=self.entries.get(key) if self.enabled else None
|
|
117
|
+
if cached and (cached[1] or len(cached[0])>=limit):
|
|
118
|
+
self.hits+=1;self.entries.move_to_end(key);return cached[0][:limit]
|
|
119
|
+
remaining=self.max_bytes-self.bytes_read if self.max_bytes is not None else limit+1
|
|
120
|
+
if remaining<=0:
|
|
121
|
+
self.failures['read_budget']+=1;return b''
|
|
122
|
+
read_limit=min(limit+1,remaining)
|
|
123
|
+
try:
|
|
124
|
+
with self._open(key) as stream:raw=stream.read(read_limit)
|
|
125
|
+
except OSError:self.failures['unreadable_or_symlink']+=1;return b''
|
|
126
|
+
self.reads+=1;self.bytes_read+=len(raw);complete=len(raw)<=limit and not (read_limit<limit+1 and len(raw)==read_limit)
|
|
127
|
+
if not complete and read_limit<limit+1:self.failures['read_budget']+=1
|
|
128
|
+
raw=raw[:limit]
|
|
129
|
+
self.evidence[key]={'path':key,'sha256':hashlib.sha256(raw).hexdigest(),'complete':complete,'bytes':len(raw)}
|
|
130
|
+
old=self.entries.pop(key,None)
|
|
131
|
+
if old:self.size-=len(old[0])
|
|
132
|
+
if self.enabled and len(raw)<=CACHE_BYTES:
|
|
133
|
+
self.entries[key]=(raw,complete);self.size+=len(raw)
|
|
134
|
+
while self.size>CACHE_BYTES:
|
|
135
|
+
_,value=self.entries.popitem(last=False);self.size-=len(value[0])
|
|
136
|
+
return raw
|
|
137
|
+
|
|
138
|
+
def text(self,path,limit=1500001):return self.read(path,limit).decode('utf8','ignore')
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def folder_reason(relative,name,families,names=None):
|
|
142
|
+
parts=PurePosixPath(relative).parts
|
|
143
|
+
if name in HARD_SKIP:return 'vcs_metadata'
|
|
144
|
+
if name in DEPENDENCIES or name in {'.venv','venv'}:return 'dependency_installation'
|
|
145
|
+
if re.fullmatch(r'(?:pyodide|monaco_support)(?:_[A-Za-z0-9.-]+)?',name):return 'browser_runtime_dependency'
|
|
146
|
+
if name in {'Pods','DerivedData','.dart_tool'}:return 'platform_dependency_or_generated'
|
|
147
|
+
if name.startswith('cmake-build-'):return 'native_generated_output'
|
|
148
|
+
if name in GENERATED or name.startswith('.cache') or name in {'.pytest_cache','.ruff_cache','.mypy_cache','.nox','.idea','.codex-logs'}:return 'generated_or_cache'
|
|
149
|
+
if '.yarn' in parts and any(p in {'cache','unplugged','sdks'} for p in parts[parts.index('.yarn')+1:]):return 'node_dependency_store'
|
|
150
|
+
if '.cargo' in parts and any(p in {'registry','git','bin'} for p in parts[parts.index('.cargo')+1:]):return 'rust_dependency_toolchain'
|
|
151
|
+
if name in {'bin','obj'} and 'dotnet' in families:return 'dotnet_generated_output'
|
|
152
|
+
if name in {'archives','archive','backups','backup','snapshots','__MACOSX','.tmp-import'}:return 'archive_or_metadata'
|
|
153
|
+
if any(re.search(r'(?i)(?:canary-source|release-stage|windows-release|staging-copy)',p) for p in parts):return 'generated_project_copy'
|
|
154
|
+
if name in {'submissions','uploaded_zip_contents','dropins'}:return 'submitted_project_corpus'
|
|
155
|
+
if name in {'wp-admin','wp-includes'} and 'wordpress' in families:return 'wordpress_core_dependency'
|
|
156
|
+
if 'wordpress' in families and len(parts)>=2 and parts[-2]=='wp-content' and name in {'plugins','themes','uploads'}:return 'wordpress_deployed_package'
|
|
157
|
+
if name in {'extensions','plugins'} and names and any(n.endswith('.jar') for n in names):return 'installed_extension_cache'
|
|
158
|
+
if names:
|
|
159
|
+
if 'packages.xml' in names and {'build-tools','platform-tools','platforms'} & set(names):return 'android_sdk_toolchain'
|
|
160
|
+
if 'pyvenv.cfg' in names or (re.match(r'^\.?venv[_-]',name) and {'Lib','lib','Scripts','bin'} & set(names)):return 'python_environment'
|
|
161
|
+
if name=='packages' and any(n.endswith(('.nupkg','.nuspec')) for n in names):return 'restored_nuget_packages'
|
|
162
|
+
if 'packages' in parts[:-1] and re.search(r'\.\d+\.\d+',name) and any(n.endswith('.nuspec') for n in names) and not any(n.endswith('.csproj') for n in names):return 'restored_nuget_packages'
|
|
163
|
+
return None
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def file_role(relative):
|
|
167
|
+
p=PurePosixPath(relative);parts=p.parts
|
|
168
|
+
if p.name in METADATA or p.name.startswith('._') or '__MACOSX' in parts:return 'metadata'
|
|
169
|
+
if p.name=='.gitignore':return 'metadata'
|
|
170
|
+
if p.suffix.lower()=='.swift' and any(x.lower()=='tests' for x in parts[:-1]):return 'tests'
|
|
171
|
+
if any(x in {'tests','test','__tests__'} for x in parts[:-1]) or re.search(r'(^test_|_test\.|\.test\.|\.spec\.)',p.name):return 'tests'
|
|
172
|
+
if any(x in {'fixtures','__fixtures__'} for x in parts[:-1]):return 'fixture'
|
|
173
|
+
if any(x in {'docs','references'} for x in parts[:-1]) or p.suffix.lower() in {'.md','.rst','.adoc'}:return 'docs'
|
|
174
|
+
if any(x in {'tools','scripts','artifacts','.github'} for x in parts[:-1]):return 'tooling'
|
|
175
|
+
if any(x in {'src','app','lib','apps','services','crates','modules','features','domain','application'} for x in parts[:-1]):return 'production'
|
|
176
|
+
return 'unknown'
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class Scope:
|
|
180
|
+
def __init__(self,root,respect_gitignore=True,policy='typed',config=None,inventory_all=False,max_directories=1000000,max_entries=2000000,max_depth=256,max_read_bytes=None):
|
|
181
|
+
self.root=Path(root).resolve();self.respect_gitignore=respect_gitignore;self.policy=policy
|
|
182
|
+
self.walk_limits=(max_directories,max_entries,max_depth);self.max_read_bytes=max_read_bytes
|
|
183
|
+
if policy not in {'typed','legacy'}:raise ValueError('Unknown scope policy')
|
|
184
|
+
self.config_control=getattr(config,'control',None);self.config=validate_config(config or {});self.inventory_all=inventory_all;self.cache=ReadCache(self.root,max_read_bytes)
|
|
185
|
+
self.files=[];self.records=[];self.by_path={};self.nodes=[];self.exclusions=[];self.ignore_sources=[];self.diagnostics=[]
|
|
186
|
+
self.counts=Counter();self.all_extensions=Counter();self.collected_extensions=Counter();self.role_counts=Counter();self._seen=0
|
|
187
|
+
self.graph={};self.source_plan=[];self.ambiguous_roots=[];self._ignored_rules=0
|
|
188
|
+
self.include=self.config.get('include',[]);self.exclude=self.config.get('exclude',[])
|
|
189
|
+
|
|
190
|
+
def _can_descend_include(self,relative):
|
|
191
|
+
for pattern in self.include:
|
|
192
|
+
prefix=re.split(r'[*?\[]',pattern,maxsplit=1)[0].rstrip('/')
|
|
193
|
+
if not prefix or prefix==relative or prefix.startswith(relative+'/') or relative.startswith(prefix+'/'):return True
|
|
194
|
+
return False
|
|
195
|
+
|
|
196
|
+
def _excluded(self,relative,reason,directory=False,evidence=None):
|
|
197
|
+
self.counts[('pruned_directories:' if directory else 'excluded_files:')+reason]+=1
|
|
198
|
+
if len(self.exclusions)<2000:self.exclusions.append({'path':relative,'reason':reason,'kind':'directory' if directory else 'file',**({'evidence':evidence} if evidence else {})})
|
|
199
|
+
|
|
200
|
+
def build(self):
|
|
201
|
+
stack=[(self.root,[],set(),None)]
|
|
202
|
+
while stack:
|
|
203
|
+
directory,parent_rules,parent_families,inherited=stack.pop();relative=directory.relative_to(self.root).as_posix()
|
|
204
|
+
if self.counts['directories_visited']>=self.walk_limits[0] or self.counts['entries_visited']>=self.walk_limits[1]:
|
|
205
|
+
self.diagnostics.append({'path':relative,'reason':'scope_traversal_budget'});break
|
|
206
|
+
if len(directory.relative_to(self.root).parts)>self.walk_limits[2]:
|
|
207
|
+
self.diagnostics.append({'path':relative,'reason':'scope_depth_budget'});continue
|
|
208
|
+
self.counts['directories_visited']+=1
|
|
209
|
+
try:
|
|
210
|
+
with os.scandir(directory) as scan:
|
|
211
|
+
entries=[]
|
|
212
|
+
for entry in scan:
|
|
213
|
+
if self.counts['entries_visited']>=self.walk_limits[1]:
|
|
214
|
+
self.diagnostics.append({'path':relative,'reason':'scope_traversal_budget'});break
|
|
215
|
+
entries.append(entry);self.counts['entries_visited']+=1
|
|
216
|
+
entries.sort(key=lambda e:e.name)
|
|
217
|
+
except OSError:
|
|
218
|
+
self.diagnostics.append({'path':relative,'reason':'unreadable_directory'});continue
|
|
219
|
+
names={e.name for e in entries};rules=parent_rules;families=set(parent_families)
|
|
220
|
+
if relative!='.' and self.policy=='typed' and not inherited:
|
|
221
|
+
why=folder_reason(relative,directory.name,families,names)
|
|
222
|
+
if why and not self._can_descend_include(relative):
|
|
223
|
+
self._excluded(relative,why,True)
|
|
224
|
+
if not self.inventory_all:continue
|
|
225
|
+
inherited=why
|
|
226
|
+
if self.respect_gitignore and '.gitignore' in names and (not inherited or (inherited not in {'gitignore','operator_exclude','vcs_metadata'} and self._can_descend_include(relative))):
|
|
227
|
+
control=directory/'.gitignore'
|
|
228
|
+
if not control.is_symlink():
|
|
229
|
+
before_failures=sum(self.cache.failures.values())
|
|
230
|
+
raw=self.cache.read(control,MAX_IGNORE_BYTES+1);rp=control.relative_to(self.root).as_posix()
|
|
231
|
+
if sum(self.cache.failures.values())!=before_failures:raise ValueError('Unreadable .gitignore; refusing to broaden scope: '+rp)
|
|
232
|
+
parsed,error=parse_ignore(raw,rp)
|
|
233
|
+
if self._ignored_rules+len(parsed)>MAX_RULES:error='gitignore_rule_limit';parsed=[]
|
|
234
|
+
self._ignored_rules+=len(parsed);rules=parent_rules+parsed;self.ignore_sources.append(ignore_evidence(raw,rp))
|
|
235
|
+
if error:raise ValueError('Cannot safely apply '+rp+': '+error+'; review this ignore file or explicitly use --no-gitignore')
|
|
236
|
+
visible_names=[]
|
|
237
|
+
for entry in entries:
|
|
238
|
+
rp=entry.name if relative=='.' else relative+'/'+entry.name
|
|
239
|
+
ignored,_=ignore_decision(rp,entry.is_dir(follow_symlinks=False),rules) if self.respect_gitignore else (False,None)
|
|
240
|
+
if not ignored and not entry.is_symlink():visible_names.append(entry.name)
|
|
241
|
+
node=None
|
|
242
|
+
if (not inherited or (inherited not in {'gitignore','operator_exclude','vcs_metadata'} and matches(relative,self.include))) and len(self.nodes)<MAX_PROJECTS and self.counts['manifests_read']<MAX_MANIFESTS:
|
|
243
|
+
node=node_for(directory,relative,visible_names,self.cache.text,root=relative=='.')
|
|
244
|
+
if node and (relative=='.' or not archived(relative)):
|
|
245
|
+
# Marker plus source/entrypoint proximity; avoid package metadata alone.
|
|
246
|
+
credible=relative=='.' or bool(set(visible_names)&{'src','app','lib','index.js','main.py','main.go','Program.cs','main.rs'}) or any(Path(n).suffix.lower() in LANGUAGES for n in visible_names)
|
|
247
|
+
self.counts['manifests_read']+=len(node['evidence'])
|
|
248
|
+
if credible:
|
|
249
|
+
self.nodes.append(node);families.update(node['families'])
|
|
250
|
+
elif len(self.ambiguous_roots)<512:self.ambiguous_roots.append({'root':relative,'families':node['families'],'evidence':node['evidence'],'reason':'marker_without_nearby_source','confidence':'low'})
|
|
251
|
+
elif not inherited:self.diagnostics.append({'path':relative,'reason':'project_discovery_limit'})
|
|
252
|
+
children=[]
|
|
253
|
+
for entry in entries:
|
|
254
|
+
path=Path(entry.path);rp=path.relative_to(self.root).as_posix()
|
|
255
|
+
try:info=entry.stat(follow_symlinks=False)
|
|
256
|
+
except OSError:self._excluded(rp,'unreadable_metadata');continue
|
|
257
|
+
if stat.S_ISLNK(info.st_mode):self.counts['symlinks_skipped']+=1;continue
|
|
258
|
+
is_dir=stat.S_ISDIR(info.st_mode)
|
|
259
|
+
ignored,ignore_ref=ignore_decision(rp,is_dir,rules) if self.respect_gitignore else (False,None)
|
|
260
|
+
reason=inherited
|
|
261
|
+
if ignored:reason='gitignore'
|
|
262
|
+
elif matches(rp,self.exclude):reason='operator_exclude'
|
|
263
|
+
elif not reason and is_dir:
|
|
264
|
+
if self.policy=='legacy':reason='legacy_directory' if entry.name in LEGACY_SKIP or entry.name.startswith('.cache') else None
|
|
265
|
+
else:reason=folder_reason(rp,entry.name,families)
|
|
266
|
+
if is_dir:
|
|
267
|
+
if reason and reason not in {'gitignore','operator_exclude','vcs_metadata'} and self._can_descend_include(rp):
|
|
268
|
+
children.append((path,rules,families,reason));continue
|
|
269
|
+
if reason:
|
|
270
|
+
self._excluded(rp,reason,True,ignore_ref)
|
|
271
|
+
if self.inventory_all:children.append((path,rules,families,reason))
|
|
272
|
+
else:children.append((path,rules,families,None))
|
|
273
|
+
continue
|
|
274
|
+
if not stat.S_ISREG(info.st_mode):self._excluded(rp,'unsupported_file_type');continue
|
|
275
|
+
self._seen+=1;self.all_extensions[path.suffix.lower()]+=1
|
|
276
|
+
included=matches(rp,self.include)
|
|
277
|
+
if included and reason not in {'gitignore','operator_exclude','vcs_metadata'}:reason=None
|
|
278
|
+
role=file_role(rp) if self.policy=='typed' else 'unknown'
|
|
279
|
+
for pattern,override in self.config.get('roles',{}).items():
|
|
280
|
+
if matches(rp,[pattern]):role=override
|
|
281
|
+
if not reason and self.respect_gitignore and entry.name=='.gitignore':reason='ignore_control_metadata'
|
|
282
|
+
if not reason and self.policy=='typed' and role=='metadata':reason='os_metadata'
|
|
283
|
+
if not reason and self.policy=='typed' and entry.name in {'.pnp.cjs','.pnp.loader.mjs'} and 'node' in families and not included:reason='node_generated_installation'
|
|
284
|
+
if not reason and self.policy=='typed' and (entry.name.endswith('.min.js') or entry.name.endswith('.min.css')) and not included:reason='generated_minified'
|
|
285
|
+
if not reason and self.policy=='typed' and role=='fixture' and not included:reason='fixture_corpus'
|
|
286
|
+
if not reason and info.st_size>1500000:reason='core_file_size_limit'
|
|
287
|
+
if not reason and len(self.records)>=MAX_FILES:reason='inventory_file_limit'
|
|
288
|
+
if reason:self._excluded(rp,reason,evidence=ignore_ref);continue
|
|
289
|
+
record={'path':path,'relative':rp,'size':info.st_size,'mtime_ns':info.st_mtime_ns,'role':role,
|
|
290
|
+
'owner':'root','language':LANGUAGES.get(path.suffix.lower(),'other'),'priority':(2,rp)}
|
|
291
|
+
record['authorship']='operator_selected' if included else 'unknown'
|
|
292
|
+
if not path.suffix and info.st_size<=1500000:
|
|
293
|
+
from .adapters import shebang_language
|
|
294
|
+
inferred=shebang_language(self.cache.text(path,256))
|
|
295
|
+
if inferred:record['language']=inferred;record['language_basis']='bounded_shebang'
|
|
296
|
+
self.records.append(record);self.by_path[path]=record;self.collected_extensions[path.suffix.lower()]+=1;self.role_counts[role]+=1
|
|
297
|
+
stack.extend(reversed(children))
|
|
298
|
+
if not self.nodes:self.nodes=[node_for(self.root,'.',[],self.cache.text,True)]
|
|
299
|
+
for declared in self.config.get('projects',[]):
|
|
300
|
+
relative=declared['root'];path=self.root/relative
|
|
301
|
+
if not self._safe_path(path) or not path.is_dir():raise ValueError('Declared project must be an existing non-symlink root inside scan')
|
|
302
|
+
existing=next((n for n in self.nodes if n['root']==relative),None)
|
|
303
|
+
if existing is None:
|
|
304
|
+
existing=node_for(path,relative,[],self.cache.text,True);existing['id']=__import__('aqaf.projects',fromlist=['project_id']).project_id(relative);existing['root']=relative;self.nodes.append(existing)
|
|
305
|
+
existing['relationship']='operator_declared';existing['confidence']='user_selected';existing['profile_override']=declared.get('profile')
|
|
306
|
+
existing['source_hints'].extend({'path':x,'kind':'operator_declared'} for x in declared.get('source_roots',[]))
|
|
307
|
+
self.graph=graph(self.nodes,self.records,self.root)
|
|
308
|
+
self.graph['ambiguous_roots']=self.ambiguous_roots
|
|
309
|
+
self._priority_plan()
|
|
310
|
+
self.records.sort(key=lambda r:r['relative'] if self.policy=='legacy' else r['priority'])
|
|
311
|
+
self.files=[r['path'] for r in self.records]
|
|
312
|
+
return self
|
|
313
|
+
|
|
314
|
+
def _safe_path(self,path):
|
|
315
|
+
try:relative=path.relative_to(self.root)
|
|
316
|
+
except ValueError:return False
|
|
317
|
+
candidate=self.root
|
|
318
|
+
for part in relative.parts:
|
|
319
|
+
candidate/=part
|
|
320
|
+
if part=='..' or candidate.is_symlink():return False
|
|
321
|
+
return True
|
|
322
|
+
|
|
323
|
+
def _priority_plan(self):
|
|
324
|
+
for node in self.nodes:
|
|
325
|
+
prefix='' if node['root']=='.' else node['root']+'/'
|
|
326
|
+
for hint in node['source_roots']:
|
|
327
|
+
rp=prefix+hint['path'];self.source_plan.append({'path':rp,'owner':node['id'],**{k:v for k,v in hint.items() if k!='path'}})
|
|
328
|
+
for rp in self.config.get('source_roots',[]):
|
|
329
|
+
path=self.root/rp
|
|
330
|
+
if not self._safe_path(path):raise ValueError('Unsafe source priority path')
|
|
331
|
+
self.source_plan.append({'path':rp,'owner':'root','kind':'operator_declared','confidence':'user_selected','exists':path.exists()})
|
|
332
|
+
if not path.exists():self.diagnostics.append({'path':rp,'reason':'stale_operator_source_root'})
|
|
333
|
+
# Only bounded architecture/ADR documents provide weak path hints, never instructions.
|
|
334
|
+
documents=[r for r in self.records if r['path'].suffix.lower() in {'.md','.rst','.adoc'} and re.search(r'(?i)architecture|(?:^|/)adr(?:/|$)',r['relative'])]
|
|
335
|
+
for record in documents[:64]:
|
|
336
|
+
raw=self.cache.read(record['path'],32768);text=raw.decode('utf8','ignore')
|
|
337
|
+
for candidate in re.findall(r'`([^`\n]{1,128})`',text)[:64]:
|
|
338
|
+
if not safe_relative(candidate) or any(c in candidate for c in '*? $;&|<>'):continue
|
|
339
|
+
path=self.root/candidate
|
|
340
|
+
if not self._safe_path(path):continue
|
|
341
|
+
if path.exists():self.source_plan.append({'path':candidate,'owner':record['owner'],'kind':'architecture_hint','confidence':'low','evidence':record['relative'],'sha256':hashlib.sha256(raw).hexdigest()})
|
|
342
|
+
elif '/' in candidate:self.diagnostics.append({'path':candidate,'reason':'stale_architecture_hint','evidence':record['relative']})
|
|
343
|
+
manifest_evidence={e['path']:e['sha256'] for node in self.nodes for e in node['evidence']}
|
|
344
|
+
config_digest=hashlib.sha256(json.dumps(self.config,sort_keys=True).encode()).hexdigest()
|
|
345
|
+
for hint in self.source_plan:
|
|
346
|
+
if hint['kind']=='operator_declared':hint['effective_config_sha256']=config_digest
|
|
347
|
+
elif hint.get('evidence') in manifest_evidence:hint['sha256']=manifest_evidence[hint['evidence']]
|
|
348
|
+
elif hint['kind']=='conventional':hint['evidence']='conventional_layout_fallback'
|
|
349
|
+
for r in self.records:
|
|
350
|
+
hints=[h for h in self.source_plan if r['relative']==h['path'] or r['relative'].startswith(h['path'].rstrip('/')+'/')]
|
|
351
|
+
rank=min((0 if h['kind']=='operator_declared' else 1 if h['kind']=='entrypoint' else 2 if h['kind']=='architecture_hint' else 3 for h in hints),default=4)
|
|
352
|
+
r['priority']=(rank,ROLE_ORDER.get(r['role'],2),r['relative'])
|
|
353
|
+
if hints:r['priority_evidence']=[{'path':h['path'],'kind':h['kind']} for h in hints[:8]]
|
|
354
|
+
|
|
355
|
+
def selection_complete(self):
|
|
356
|
+
incomplete={'unreadable_directory','scope_traversal_budget','scope_depth_budget','project_discovery_limit'}
|
|
357
|
+
return not self.cache.failures and not any(d['reason'] in incomplete for d in self.diagnostics) and not self.counts['excluded_files:unreadable_metadata'] and not self.counts['excluded_files:inventory_file_limit']
|
|
358
|
+
|
|
359
|
+
def selected(self,roles=None):
|
|
360
|
+
"""Version 1 selection shared by code, documentation and discovery."""
|
|
361
|
+
return [r for r in self.records if roles is None or r['role'] in roles]
|
|
362
|
+
|
|
363
|
+
def provenance(self):
|
|
364
|
+
return {'schema_version':1,'selection':self.report(),'read_inputs':sorted(self.cache.evidence.values(),key=lambda r:r['path'])}
|
|
365
|
+
|
|
366
|
+
def record(self,path):return self.by_path.get(Path(path),{})
|
|
367
|
+
|
|
368
|
+
def text(self,path,limit=1500001):
|
|
369
|
+
if Path(path) not in self.by_path:return ''
|
|
370
|
+
return self.cache.text(path,limit)
|
|
371
|
+
|
|
372
|
+
def raw(self,path,limit=1500001):
|
|
373
|
+
if Path(path) not in self.by_path:return b''
|
|
374
|
+
return self.cache.read(path,limit)
|
|
375
|
+
|
|
376
|
+
def report(self):
|
|
377
|
+
import unicodedata
|
|
378
|
+
normalized=defaultdict(list)
|
|
379
|
+
for record in self.records:normalized[unicodedata.normalize('NFC',record['relative'])].append(record['relative'])
|
|
380
|
+
collisions=[{'display_path':key,'original_paths':values} for key,values in sorted(normalized.items()) if len(values)>1]
|
|
381
|
+
exclusions=[]
|
|
382
|
+
for entry in self.exclusions:
|
|
383
|
+
candidates=[n for n in self.nodes if n['root']=='.' or entry['path']==n['root'] or entry['path'].startswith(n['root']+'/')]
|
|
384
|
+
owner=max(candidates,key=lambda n:(n['root']!='.',len(n['root']))) if candidates else self.nodes[0]
|
|
385
|
+
reason=entry['reason']
|
|
386
|
+
role='metadata' if reason in {'os_metadata','ignore_control_metadata','vcs_metadata'} or entry['path'].split('/')[0]=='__MACOSX' else 'archive' if reason=='archive_or_metadata' else 'dependency' if any(s in reason for s in ['dependency','toolchain','environment','nuget']) else 'generated' if any(s in reason for s in ['generated','cache','minified']) else 'fixture' if 'fixture' in reason or 'corpus' in reason else 'excluded_unknown'
|
|
387
|
+
exclusions.append({**entry,'role':role,'owner':owner['id'],'confidence':'operator_selected' if reason=='operator_exclude' else 'heuristic_policy',
|
|
388
|
+
'policy_evidence':{'policy':self.policy,'owner_families':owner['families'],'reason_code':reason}})
|
|
389
|
+
return {'selection_complete':self.selection_complete(),'operator_control':self.config_control,'selection_api_version':1,'unicode_display_collisions':collisions[:1000],'effective_config_sha256':hashlib.sha256(json.dumps(self.config,sort_keys=True).encode()).hexdigest(),'schema_version':1,'policy':self.policy,'respect_gitignore':self.respect_gitignore,
|
|
390
|
+
'gitignore_sources':self.ignore_sources,'gitignore_tracked_status':'not_consulted_patterns_apply_to_all_paths',
|
|
391
|
+
'gitignore_scope':'authorized_root_and_descendants_only','physical_census_complete':self.inventory_all and not self.diagnostics and not self.counts['excluded_files:unreadable_metadata'],
|
|
392
|
+
'visited_regular_files':self._seen,'collected_files':len(self.files),'visited_extensions':dict(sorted(self.all_extensions.items())),
|
|
393
|
+
'collected_extensions':dict(sorted(self.collected_extensions.items())),'roles':dict(sorted(self.role_counts.items())),
|
|
394
|
+
'counts':dict(sorted(self.counts.items())),'exclusions':exclusions,'exclusion_details_truncated':sum(v for k,v in self.counts.items() if k.startswith(('pruned_directories:','excluded_files:')))>len(self.exclusions),
|
|
395
|
+
'diagnostics':self.diagnostics[:2000],'source_plan':self.source_plan[:2000],
|
|
396
|
+
'limits':{'max_directories':self.walk_limits[0],'max_entries':self.walk_limits[1],'max_depth':self.walk_limits[2],'max_read_bytes':self.max_read_bytes,'max_selected_files':MAX_FILES,'max_projects':MAX_PROJECTS,'max_manifest_bytes':131072,'source_cache_bytes':CACHE_BYTES},
|
|
397
|
+
'read_cache':{'hits':self.cache.hits,'file_reads':self.cache.reads,'bytes_read':self.cache.bytes_read,'failures':dict(self.cache.failures)},
|
|
398
|
+
'limitations':['Ownership and architecture are inferred from bounded static evidence, not verified intent.',
|
|
399
|
+
'Pruned subtrees have unknown descendant counts unless inventory_all was selected.',
|
|
400
|
+
'Source is read once per cache entry, not an atomic whole-repository snapshot; kit integrity is separate.',
|
|
401
|
+
'Global gitignore, Git index tracked status, .git/info/exclude and patterns outside the authorized root are not read.']}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Passive Swift lexical/source facts. No compiler, imports or macro expansion."""
|
|
2
|
+
import re
|
|
3
|
+
from .adapters import Emitter,_pairs,MAX_SIGNALS
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def mask_swift(text):
|
|
7
|
+
chars=list(text);n=len(text);i=0;diagnostics=set()
|
|
8
|
+
def erase(a,b):
|
|
9
|
+
for k in range(a,b):
|
|
10
|
+
if chars[k] not in '\r\n':chars[k]=' '
|
|
11
|
+
def comment(pos):
|
|
12
|
+
level=1;pos+=2
|
|
13
|
+
while pos<n and level:
|
|
14
|
+
if text.startswith('/*',pos):level+=1;pos+=2
|
|
15
|
+
elif text.startswith('*/',pos):level-=1;pos+=2
|
|
16
|
+
else:pos+=1
|
|
17
|
+
if level>64:raise ValueError('unsupported_limit')
|
|
18
|
+
if level:raise ValueError('unsupported_syntax')
|
|
19
|
+
return pos
|
|
20
|
+
def interpolation(pos,depth):
|
|
21
|
+
level=1
|
|
22
|
+
while pos<n:
|
|
23
|
+
if text.startswith('/*',pos):pos=comment(pos);continue
|
|
24
|
+
if text.startswith('//',pos):
|
|
25
|
+
end=text.find('\n',pos);pos=n if end<0 else end;continue
|
|
26
|
+
m=re.match(r'(#{0,16})("""|")',text[pos:])
|
|
27
|
+
if m:pos=string(pos,m,depth+1);continue
|
|
28
|
+
if text[pos]=='(':level+=1
|
|
29
|
+
elif text[pos]==')':
|
|
30
|
+
level-=1
|
|
31
|
+
if not level:return pos+1
|
|
32
|
+
pos+=1
|
|
33
|
+
raise ValueError('unsupported_syntax')
|
|
34
|
+
def string(pos,match,depth):
|
|
35
|
+
if depth>64:raise ValueError('unsupported_limit')
|
|
36
|
+
hashes,quote=match.groups();end_delimiter=quote+hashes;escape='\\'+hashes;pos+=len(match.group())
|
|
37
|
+
while pos<n:
|
|
38
|
+
if text.startswith(escape+'(',pos):
|
|
39
|
+
diagnostics.add('swift_string_interpolation_unassessed');pos=interpolation(pos+len(escape)+1,depth);continue
|
|
40
|
+
if text.startswith(escape,pos):pos+=len(escape)+1;continue
|
|
41
|
+
if text.startswith(end_delimiter,pos):return pos+len(end_delimiter)
|
|
42
|
+
if len(quote)==1 and text[pos] in '\r\n':raise ValueError('unsupported_syntax')
|
|
43
|
+
pos+=1
|
|
44
|
+
raise ValueError('unsupported_syntax')
|
|
45
|
+
while i<n:
|
|
46
|
+
start=i
|
|
47
|
+
if text.startswith('//',i):
|
|
48
|
+
end=text.find('\n',i);i=n if end<0 else end;erase(start,i);continue
|
|
49
|
+
if text.startswith('/*',i):i=comment(i);erase(start,i);continue
|
|
50
|
+
match=re.match(r'(#{0,16})("""|")',text[i:]) if text[i] in '#"' else None
|
|
51
|
+
if match:i=string(i,match,0);erase(start,i);continue
|
|
52
|
+
if text.startswith('#/',i):raise ValueError('unsupported_syntax')
|
|
53
|
+
if text[i]=='/' and re.search(r'(?:^|[=(:,]|\breturn)\s*$',text[:i]):raise ValueError('unsupported_syntax')
|
|
54
|
+
i+=1
|
|
55
|
+
return ''.join(chars),sorted(diagnostics)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def swift_review(text,path,context):
|
|
59
|
+
emitter=Emitter(text,path,'swift',context)
|
|
60
|
+
try:
|
|
61
|
+
mask,diagnostics=mask_swift(text);pairs=_pairs(mask)
|
|
62
|
+
if len(re.findall(r'\w+|[^\s\w]',mask))>20000:return {'status':'unsupported_limit','signals':[]}
|
|
63
|
+
except ValueError as error:return {'status':str(error),'signals':[]}
|
|
64
|
+
def block(index):
|
|
65
|
+
candidates=[(a,b) for a,b in pairs.items() if a<index<b and mask[a]=='{']
|
|
66
|
+
return max(candidates,key=lambda p:p[0]) if candidates else (0,len(text))
|
|
67
|
+
def shadowed(name):return bool(re.search(r'\b(?:class|struct|enum|typealias|func)\s+'+name+r'\b',mask))
|
|
68
|
+
imported=bool(re.search(r'\bimport\s+(?:Foundation|Cocoa|AppKit|SwiftUI)\b',mask))
|
|
69
|
+
if imported:
|
|
70
|
+
for match in re.finditer(r'\b(?:Foundation\.)?Process\s*\(\s*\)',mask):
|
|
71
|
+
if not match.group().startswith('Foundation.') and shadowed('Process'):continue
|
|
72
|
+
a,b=block(match.start());prefix=mask[a:match.start()]
|
|
73
|
+
binding=re.search(r'\b(?:let|var)\s+([A-Za-z_]\w*)\s*(?::\s*(?:Foundation\.)?Process\??\s*)?=\s*$',prefix)
|
|
74
|
+
controls=[]
|
|
75
|
+
if binding:
|
|
76
|
+
name=re.escape(binding.group(1))
|
|
77
|
+
for control in re.finditer(r'\b'+name+r'\s*(?:\?\.|\.)\s*(terminationHandler\s*=|waitUntilExit\s*\(|terminate\s*\()',mask[match.end():b]):
|
|
78
|
+
index=match.end()+control.start()
|
|
79
|
+
if block(index)==(a,b):controls.append('A '+control.group(1).split('(')[0].strip(' =')+' spelling is visible on the same local binding.')
|
|
80
|
+
emitter.emit('AQR-LIFE-001','swift_process_controls' if controls else 'swift_process_context',match.start(),
|
|
81
|
+
'Foundation Process construction syntax is present; launch intent, transferred ownership, execution and complete cleanup remain unresolved.',
|
|
82
|
+
'observed_control' if controls else 'source_observation',controls)
|
|
83
|
+
for match in re.finditer(r'\bJSONDecoder(?:\s*\(\s*\)|\.[A-Za-z_]\w*)\s*\.\s*decode\s*\(',mask):
|
|
84
|
+
if shadowed('JSONDecoder'):continue
|
|
85
|
+
emitter.emit('AQR-VALID-001','swift_decoding_context',match.start(),
|
|
86
|
+
'JSONDecoder-shaped decoding syntax is present; Decodable typing does not establish complete domain validation or an unsafe input boundary.','source_observation')
|
|
87
|
+
for match in re.finditer(r'\b([A-Za-z_]\w*)\s*\.\s*write\s*\(\s*to\s*:',mask):
|
|
88
|
+
opening=mask.find('(',match.start());closing=pairs.get(opening)
|
|
89
|
+
if closing is None:continue
|
|
90
|
+
arguments=mask[opening:closing+1];atomic=bool(re.search(r'\boptions\s*:[^)]*\.atomic\b',arguments))
|
|
91
|
+
emitter.emit('AQR-STATE-001','swift_atomic_option' if atomic else 'swift_write_context',match.start(),
|
|
92
|
+
'A write(to:) spelling is present; receiver type, persistent-state intent and caller/framework publication controls require review.',
|
|
93
|
+
'observed_control' if atomic else 'source_observation',
|
|
94
|
+
['An .atomic option spelling is visible; filesystem semantics and full durability are not established.'] if atomic else [])
|
|
95
|
+
if not shadowed('Task'):
|
|
96
|
+
for match in re.finditer(r'\bTask(?:\s*\.\s*detached)?\s*(?:\([^{}\n]*\))?\s*\{',mask):
|
|
97
|
+
emitter.emit('AQR-LIFE-001','swift_task_context',match.start(),
|
|
98
|
+
'Task startup syntax is present; actor inheritance, captured state, retained handles, cooperative cancellation and all exit paths remain unresolved.','source_observation')
|
|
99
|
+
return {'status':'assessed_limited','signals':emitter.signals,'diagnostics':diagnostics+['swift_bounded_source_facts_not_compiler_type_or_actor_analysis']}
|