@aqaf/cli 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -3
- package/index.js +20 -1
- package/package.json +3 -2
- package/vendor/aqaf/__init__.py +1 -1
- package/vendor/aqaf/adapters.py +583 -0
- package/vendor/aqaf/artifacts.py +28 -0
- package/vendor/aqaf/calibration.py +29 -0
- package/vendor/aqaf/catalogue.py +257 -0
- package/vendor/aqaf/cli.py +171 -33
- package/vendor/aqaf/cursor.py +56 -0
- package/vendor/aqaf/data/book_review_knowledge.json +1574 -0
- package/vendor/aqaf/data/documentation_checks.json +52 -0
- package/vendor/aqaf/data/review_checks.json +319 -0
- package/vendor/aqaf/data/source_registry.json +513 -1
- package/vendor/aqaf/discovery.py +580 -0
- package/vendor/aqaf/documentation.py +210 -0
- package/vendor/aqaf/evidence.py +132 -0
- package/vendor/aqaf/facets.py +63 -0
- package/vendor/aqaf/ignore.py +109 -0
- package/vendor/aqaf/knowledge.py +110 -0
- package/vendor/aqaf/markup.py +96 -0
- package/vendor/aqaf/matching.py +22 -0
- package/vendor/aqaf/portfolio.py +145 -0
- package/vendor/aqaf/project_commands.py +132 -0
- package/vendor/aqaf/projects.py +157 -0
- package/vendor/aqaf/registry.py +4 -3
- package/vendor/aqaf/review.py +446 -0
- package/vendor/aqaf/scope.py +401 -0
- package/vendor/aqaf/swift_review.py +99 -0
|
@@ -0,0 +1,580 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Find project roots and estimate their types without running target code."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from collections import Counter
|
|
7
|
+
import fnmatch
|
|
8
|
+
import hashlib
|
|
9
|
+
import importlib
|
|
10
|
+
import io
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
import re
|
|
15
|
+
import sys
|
|
16
|
+
import tokenize
|
|
17
|
+
|
|
18
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
19
|
+
TOOL_VERSION = '1.0.0'
|
|
20
|
+
OS_SKIP = {'.Trash', '.Trashes', '.Spotlight-V100', '.fseventsd',
|
|
21
|
+
'$RECYCLE.BIN', 'System Volume Information', '.ssh', '.gnupg', '.credentials'}
|
|
22
|
+
ARCHIVE_REASONS = {'archive_or_metadata', 'submitted_project_corpus', 'generated_project_copy'}
|
|
23
|
+
EXTRA_MARKERS = {'requirements.txt': 'python', 'Pipfile': 'python',
|
|
24
|
+
'Gemfile': 'ruby', '*.xcodeproj': 'swift', '*.xcworkspace': 'swift'}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load_engine(source=None):
|
|
28
|
+
"""Use the installed trusted runtime; never import a searched project."""
|
|
29
|
+
from . import __version__
|
|
30
|
+
modules = {name: importlib.import_module('aqaf.' + name)
|
|
31
|
+
for name in ('projects','scope','ignore','registry','adapters','review')}
|
|
32
|
+
modules.update(source=ROOT, version=__version__)
|
|
33
|
+
return modules
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def known_projects(registry, history):
|
|
37
|
+
"""Previous detections are context, never independently verified type labels."""
|
|
38
|
+
known = {}
|
|
39
|
+
if registry and registry.is_dir():
|
|
40
|
+
for p in sorted(registry.iterdir()):
|
|
41
|
+
if p.is_symlink():
|
|
42
|
+
continue
|
|
43
|
+
if p.is_dir() and not p.name.startswith('.'):
|
|
44
|
+
known[str(p.resolve())] = {'registered_id': p.name}
|
|
45
|
+
elif p.is_file() and p.name.endswith('.aqaf-source.json') and p.stat().st_size <= 16384:
|
|
46
|
+
try:
|
|
47
|
+
d = json.loads(p.read_text())
|
|
48
|
+
if (d.get('schema_version') == 1 and d.get('kind') == 'aqaf-source-reference'
|
|
49
|
+
and isinstance(d.get('path'), str) and Path(d['path']).is_absolute()):
|
|
50
|
+
known[str(Path(d['path']))] = {'registered_id': d.get('id')}
|
|
51
|
+
except (OSError, ValueError, TypeError):
|
|
52
|
+
continue
|
|
53
|
+
if history:
|
|
54
|
+
try:
|
|
55
|
+
d = json.loads((history / 'summary.json').read_text())
|
|
56
|
+
if d.get('status') == 'passed':
|
|
57
|
+
for ident, value in d.get('projects', {}).items():
|
|
58
|
+
path = d.get('source_paths', {}).get(ident)
|
|
59
|
+
if path and value.get('status') == 'passed':
|
|
60
|
+
known.setdefault(path, {}).update(previous_profile=value.get('profile'),
|
|
61
|
+
previous_version=d.get('version'), previous_run=str(history),
|
|
62
|
+
previous_type_independently_verified=False)
|
|
63
|
+
except (OSError, ValueError, TypeError):
|
|
64
|
+
pass
|
|
65
|
+
return known
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def code_view(text, language, engine):
|
|
69
|
+
"""Keep source tokens, never classify from a copied string/comment example."""
|
|
70
|
+
if language == 'python':
|
|
71
|
+
chars = ['\n' if c == '\n' else ' ' for c in text]
|
|
72
|
+
lines = text.splitlines(keepends=True)
|
|
73
|
+
starts, offset = [], 0
|
|
74
|
+
for line in lines:
|
|
75
|
+
starts.append(offset)
|
|
76
|
+
offset += len(line)
|
|
77
|
+
try:
|
|
78
|
+
for token in tokenize.generate_tokens(io.StringIO(text).readline):
|
|
79
|
+
if token.type not in {tokenize.NAME, tokenize.OP, tokenize.NUMBER}:
|
|
80
|
+
continue
|
|
81
|
+
a = starts[token.start[0] - 1] + token.start[1]
|
|
82
|
+
b = starts[token.end[0] - 1] + token.end[1]
|
|
83
|
+
chars[a:b] = text[a:b]
|
|
84
|
+
except (tokenize.TokenError, IndentationError, IndexError, SyntaxError):
|
|
85
|
+
pass # Keep only safely tokenized code before a truncated prefix.
|
|
86
|
+
return ''.join(chars)
|
|
87
|
+
if language in {'jsx', 'tsx'}:
|
|
88
|
+
markup = importlib.import_module('aqaf.markup')
|
|
89
|
+
return code_view(markup.prepare_jsx(text)[0], 'typescript', engine)
|
|
90
|
+
if language in {'javascript', 'typescript'}:
|
|
91
|
+
tokens = engine['review'].lex(text)
|
|
92
|
+
chars = ['\n' if c == '\n' else ' ' for c in text]
|
|
93
|
+
for token in tokens:
|
|
94
|
+
if token.kind not in {'string', 'opaque'}:
|
|
95
|
+
chars[token.start:token.end] = text[token.start:token.end]
|
|
96
|
+
return ''.join(chars)
|
|
97
|
+
if language == 'swift':
|
|
98
|
+
module = importlib.import_module('aqaf.swift_review')
|
|
99
|
+
return module.mask_swift(text)[0]
|
|
100
|
+
if language == 'rust':
|
|
101
|
+
module = importlib.import_module('aqaf.evidence')
|
|
102
|
+
return module.code_tokens(text, rust=True)
|
|
103
|
+
if language not in {'python', 'kotlin', 'java', 'csharp', 'php', 'powershell',
|
|
104
|
+
'shell', 'c', 'cpp', 'c_header', 'objective_c', 'objective_cpp', 'go', 'groovy'}:
|
|
105
|
+
return '' # Other dialects retain filename/manifest evidence, never raw-text guesses.
|
|
106
|
+
return engine['adapters'].mask_source(text, language)[0]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class Finder:
|
|
110
|
+
def __init__(self, engine, *, respect_gitignore=True, max_depth=24,
|
|
111
|
+
max_directories=100000, max_entries=1000000, max_projects=2000,
|
|
112
|
+
max_files_per_project=500, max_bytes=16000000, exclude=(), known=None,
|
|
113
|
+
include_archives=False, include_installed_plugins=False, scope_config=None):
|
|
114
|
+
self.engine = engine
|
|
115
|
+
self.scope_config=scope_config or {}
|
|
116
|
+
self.registry = engine['registry'].Registry()
|
|
117
|
+
self.ignore = engine['ignore']
|
|
118
|
+
self.scope = engine['scope']
|
|
119
|
+
self.projects = engine['projects']
|
|
120
|
+
self.respect_gitignore = respect_gitignore
|
|
121
|
+
self.limits = dict(max_depth=max_depth, max_directories=max_directories,
|
|
122
|
+
max_entries=max_entries, max_projects=max_projects,
|
|
123
|
+
max_files_per_project=max_files_per_project, max_bytes=max_bytes)
|
|
124
|
+
self.exclude = tuple(exclude)+tuple(self.scope_config.get('exclude',[]))
|
|
125
|
+
self.known = known or {}
|
|
126
|
+
self.include_archives = include_archives
|
|
127
|
+
self.include_installed_plugins = include_installed_plugins
|
|
128
|
+
self.stats = Counter()
|
|
129
|
+
self.exclusions = Counter()
|
|
130
|
+
self.issues = []
|
|
131
|
+
self.rows = []
|
|
132
|
+
self.seen = set()
|
|
133
|
+
self.budget_exhausted = False
|
|
134
|
+
|
|
135
|
+
def issue(self, reason, path):
|
|
136
|
+
self.stats['issues'] += 1
|
|
137
|
+
if len(self.issues) < 100:
|
|
138
|
+
self.issues.append({'reason': reason, 'path': str(path)})
|
|
139
|
+
|
|
140
|
+
def read_raw(self, cache, path, limit):
|
|
141
|
+
remaining = self.limits['max_bytes'] - self.stats['bytes_requested']
|
|
142
|
+
if remaining <= 0:
|
|
143
|
+
if not self.budget_exhausted:
|
|
144
|
+
self.issue('content_byte_budget', cache.root)
|
|
145
|
+
self.budget_exhausted = True
|
|
146
|
+
return b''
|
|
147
|
+
allowed = min(limit, remaining)
|
|
148
|
+
raw = cache.read(path, allowed)
|
|
149
|
+
self.stats['bytes_requested'] += len(raw)
|
|
150
|
+
if allowed < limit:
|
|
151
|
+
self.issue('content_read_truncated', path)
|
|
152
|
+
return raw
|
|
153
|
+
|
|
154
|
+
def read(self, cache, path, limit):
|
|
155
|
+
return self.read_raw(cache, path, limit).decode('utf8', 'replace')
|
|
156
|
+
|
|
157
|
+
def listing(self, directory, root):
|
|
158
|
+
if self.stats['directories_visited'] >= self.limits['max_directories']:
|
|
159
|
+
self.issue('directory_budget', directory)
|
|
160
|
+
return None
|
|
161
|
+
self.stats['directories_visited'] += 1
|
|
162
|
+
entries = []
|
|
163
|
+
fd = None
|
|
164
|
+
try:
|
|
165
|
+
if (os.open in os.supports_dir_fd and os.scandir in os.supports_fd
|
|
166
|
+
and hasattr(os, 'O_NOFOLLOW')):
|
|
167
|
+
fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
168
|
+
for part in directory.relative_to(root).parts:
|
|
169
|
+
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
170
|
+
os.close(fd)
|
|
171
|
+
fd = child
|
|
172
|
+
scan = fd
|
|
173
|
+
else:
|
|
174
|
+
if any(p.is_symlink() for p in (directory, *directory.parents)):
|
|
175
|
+
raise OSError('Symlink directory rejected')
|
|
176
|
+
scan = directory
|
|
177
|
+
with os.scandir(scan) as stream:
|
|
178
|
+
for entry in stream:
|
|
179
|
+
if self.stats['entries_seen'] >= self.limits['max_entries']:
|
|
180
|
+
self.issue('entry_budget', directory)
|
|
181
|
+
return None
|
|
182
|
+
self.stats['entries_seen'] += 1
|
|
183
|
+
if len(entries) == 20000:
|
|
184
|
+
self.issue('directory_entry_limit', directory)
|
|
185
|
+
break
|
|
186
|
+
try:
|
|
187
|
+
kind = ('symlink' if entry.is_symlink() else 'directory' if entry.is_dir(follow_symlinks=False)
|
|
188
|
+
else 'file' if entry.is_file(follow_symlinks=False) else 'special')
|
|
189
|
+
entries.append((entry.name, kind))
|
|
190
|
+
except OSError:
|
|
191
|
+
self.issue('unreadable_entry', directory / entry.name)
|
|
192
|
+
except OSError:
|
|
193
|
+
self.issue('unreadable_directory', directory)
|
|
194
|
+
return None
|
|
195
|
+
finally:
|
|
196
|
+
if fd is not None:
|
|
197
|
+
os.close(fd)
|
|
198
|
+
return sorted(entries)
|
|
199
|
+
|
|
200
|
+
def markers(self, names):
|
|
201
|
+
found = {name: self.projects.marker_family(name) for name in names
|
|
202
|
+
if self.projects.marker_family(name)}
|
|
203
|
+
for pattern, family in EXTRA_MARKERS.items():
|
|
204
|
+
found.update({name: family for name in names if fnmatch.fnmatchcase(name, pattern)})
|
|
205
|
+
return found
|
|
206
|
+
|
|
207
|
+
def walk(self, root):
|
|
208
|
+
root = Path(root).expanduser().absolute()
|
|
209
|
+
if any(p.is_symlink() for p in (root, *root.parents)):
|
|
210
|
+
self.issue('search_root_symlink_rejected', root)
|
|
211
|
+
return
|
|
212
|
+
if not root.is_dir():
|
|
213
|
+
self.issue('search_root_missing_or_not_directory', root)
|
|
214
|
+
return
|
|
215
|
+
root = root.resolve()
|
|
216
|
+
cache = self.scope.ReadCache(root)
|
|
217
|
+
stack = [(root, 0, (), None, (), None)]
|
|
218
|
+
def authored(relative):
|
|
219
|
+
patterns=self.scope_config.get('include',[])
|
|
220
|
+
return any(fnmatch.fnmatchcase(relative,p) or relative==p.removesuffix('/**') for p in patterns)
|
|
221
|
+
def authored_descendant(relative):
|
|
222
|
+
return authored(relative) or any(p.startswith(relative.rstrip('/')+'/') for p in self.scope_config.get('include',[]))
|
|
223
|
+
try:
|
|
224
|
+
while stack:
|
|
225
|
+
directory, depth, rules, parent, inherited, excluded_context = stack.pop()
|
|
226
|
+
if directory in self.seen:
|
|
227
|
+
continue
|
|
228
|
+
self.seen.add(directory)
|
|
229
|
+
relative = directory.relative_to(root).as_posix()
|
|
230
|
+
if (not self.include_installed_plugins and directory != root
|
|
231
|
+
and directory.name == 'plugins' and directory.parent.name == 'wp-content'
|
|
232
|
+
and not authored_descendant(relative)):
|
|
233
|
+
self.exclusions['installed_wordpress_plugin_collection'] += 1
|
|
234
|
+
continue
|
|
235
|
+
entries = self.listing(directory, root)
|
|
236
|
+
if entries is None:
|
|
237
|
+
if self.stats['entries_seen'] >= self.limits['max_entries'] or self.stats['directories_visited'] >= self.limits['max_directories']:
|
|
238
|
+
break
|
|
239
|
+
continue
|
|
240
|
+
files, dirs = [], []
|
|
241
|
+
for name, kind in entries:
|
|
242
|
+
if kind == 'symlink': self.exclusions['symlink'] += 1
|
|
243
|
+
elif kind == 'directory': dirs.append(name)
|
|
244
|
+
elif kind == 'file': files.append(name)
|
|
245
|
+
else: self.exclusions['special_file'] += 1
|
|
246
|
+
names = set(files + dirs)
|
|
247
|
+
markers = self.markers(files + [n for n in dirs if n.endswith(('.xcodeproj', '.xcworkspace'))])
|
|
248
|
+
families = set(inherited) | set(markers.values())
|
|
249
|
+
if {'wp-content', 'wp-includes'} & names and 'wp-config.php' in names:
|
|
250
|
+
markers['wp-config.php'] = 'wordpress'
|
|
251
|
+
families.add('wordpress')
|
|
252
|
+
reason = excluded_context or self.scope.folder_reason(relative, directory.name, families, names)
|
|
253
|
+
if authored(relative) and reason!='vcs_metadata':reason=None
|
|
254
|
+
if reason=='wordpress_deployed_package' and self.include_installed_plugins and directory.name=='plugins':reason=None
|
|
255
|
+
if reason in ARCHIVE_REASONS and self.include_archives:
|
|
256
|
+
reason = None
|
|
257
|
+
if (reason and not (reason!='vcs_metadata' and authored_descendant(relative))) or directory.name in OS_SKIP or directory == ROOT / 'output':
|
|
258
|
+
self.exclusions[reason or 'private_os_or_tool_output'] += 1
|
|
259
|
+
continue
|
|
260
|
+
if self.respect_gitignore and '.gitignore' in files:
|
|
261
|
+
raw = self.read_raw(cache, directory / '.gitignore', self.ignore.MAX_IGNORE_BYTES + 1)
|
|
262
|
+
local, diagnostic = self.ignore.parse_ignore(raw, (directory / '.gitignore').relative_to(root).as_posix())
|
|
263
|
+
if diagnostic:
|
|
264
|
+
self.issue(diagnostic, directory / '.gitignore')
|
|
265
|
+
continue # Invalid controls cannot silently broaden search.
|
|
266
|
+
rules = (*rules, *local)
|
|
267
|
+
if len(rules) > self.ignore.MAX_RULES:
|
|
268
|
+
rules = rules[:self.ignore.MAX_RULES]
|
|
269
|
+
self.issue('inherited_ignore_rule_limit', directory)
|
|
270
|
+
def included(name, is_dir):
|
|
271
|
+
rel = (directory / name).relative_to(root).as_posix()
|
|
272
|
+
if any(fnmatch.fnmatchcase(rel, p) or fnmatch.fnmatchcase(name, p) for p in self.exclude):
|
|
273
|
+
self.exclusions['operator_pattern'] += 1
|
|
274
|
+
return False
|
|
275
|
+
if self.respect_gitignore and self.ignore.ignore_decision(rel, is_dir, rules)[0]:
|
|
276
|
+
self.exclusions['gitignore'] += 1
|
|
277
|
+
return False
|
|
278
|
+
return True
|
|
279
|
+
files = [name for name in files if included(name, False) and (not reason or authored((directory/name).relative_to(root).as_posix()))]
|
|
280
|
+
dirs = [name for name in dirs if included(name, True) and (not reason or authored_descendant((directory/name).relative_to(root).as_posix()))]
|
|
281
|
+
markers = {n: f for n, f in markers.items() if n in files or n in dirs}
|
|
282
|
+
if 'manifest.json' in files:
|
|
283
|
+
try:
|
|
284
|
+
manifest = json.loads(self.read(cache, directory / 'manifest.json', 65536))
|
|
285
|
+
if isinstance(manifest, dict) and 'manifest_version' in manifest:
|
|
286
|
+
markers['manifest.json'] = 'browser_extension'
|
|
287
|
+
except (ValueError, RecursionError):
|
|
288
|
+
pass
|
|
289
|
+
code = [n for n in files if Path(n).suffix.lower() in self.scope.LANGUAGES]
|
|
290
|
+
has_git = '.git' in names
|
|
291
|
+
conventional = sorted(set(dirs) & (self.projects.SOURCES | {'Sources'}))
|
|
292
|
+
# Source-only candidates remain deliberately weak; don't split every src folder.
|
|
293
|
+
nearby_source = False
|
|
294
|
+
if parent is None and not code and depth < self.limits['max_depth']:
|
|
295
|
+
for name in conventional:
|
|
296
|
+
child_entries = self.listing(directory / name, root)
|
|
297
|
+
if child_entries is None:
|
|
298
|
+
continue
|
|
299
|
+
child_files = [n for n, kind in child_entries if kind == 'file']
|
|
300
|
+
if self.markers(child_files) or '.gitignore' in child_files:
|
|
301
|
+
continue
|
|
302
|
+
nearby_source = any(Path(n).suffix.lower() in self.scope.LANGUAGES
|
|
303
|
+
and not (self.respect_gitignore and self.ignore.ignore_decision(
|
|
304
|
+
(directory / name / n).relative_to(root).as_posix(), False, rules)[0])
|
|
305
|
+
for n in child_files)
|
|
306
|
+
if nearby_source:
|
|
307
|
+
break
|
|
308
|
+
fallback = parent is None and (bool(code) or nearby_source)
|
|
309
|
+
plugin = (directory.parent.name == 'plugins' and directory.parent.parent.name == 'wp-content'
|
|
310
|
+
and any('Plugin Name:' in self.read(cache, directory / n, 8192) for n in files if n.endswith('.php')))
|
|
311
|
+
if markers or has_git or fallback or plugin:
|
|
312
|
+
if len(self.rows) >= self.limits['max_projects']:
|
|
313
|
+
self.issue('project_budget', directory)
|
|
314
|
+
break
|
|
315
|
+
row = {'path': str(directory), 'search_root': str(root), 'relative_path': relative,
|
|
316
|
+
'parent_project': parent['path'] if parent else None,
|
|
317
|
+
'relationship': 'embedded_project' if parent else 'top_level_project',
|
|
318
|
+
'markers': sorted(markers), 'ecosystems': sorted({'jvm_gradle' if f == 'android' else f for f in markers.values()}),
|
|
319
|
+
'source_roots': conventional, 'files': [], 'file_count_observed': 0,
|
|
320
|
+
'root_probability': .97 if markers and code else .90 if markers else .80 if has_git else .55,
|
|
321
|
+
'root_basis': 'manifest' if markers else 'wordpress_plugin_header' if plugin else 'git_root' if has_git else 'source_layout_only',
|
|
322
|
+
'workspace_patterns': [], 'plugin_header': plugin, 'known_context': self.known.get(str(directory)),
|
|
323
|
+
'archive_or_copy_context': self.projects.archived(directory.as_posix()),
|
|
324
|
+
'ownership': 'unverified',
|
|
325
|
+
'installed_wordpress_plugin_context': plugin,
|
|
326
|
+
'sampling_limited': False}
|
|
327
|
+
if plugin:
|
|
328
|
+
row['root_probability'] = .90
|
|
329
|
+
self.rows.append(row)
|
|
330
|
+
parent = row
|
|
331
|
+
if parent:
|
|
332
|
+
parent['file_count_observed'] += len(files)
|
|
333
|
+
candidates = [directory / n for n in files]
|
|
334
|
+
candidates.sort(key=lambda p: (not bool(self.projects.marker_family(p.name)),
|
|
335
|
+
self.scope.file_role(p.relative_to(Path(parent['path'])).as_posix()) in {'tests', 'fixture', 'docs'}, p.name))
|
|
336
|
+
remaining = self.limits['max_files_per_project'] - len(parent['files'])
|
|
337
|
+
parent['files'].extend(candidates[:remaining])
|
|
338
|
+
if len(candidates) > remaining:
|
|
339
|
+
parent['sampling_limited'] = True
|
|
340
|
+
if depth >= self.limits['max_depth']:
|
|
341
|
+
if dirs:
|
|
342
|
+
self.issue('depth_limit', directory)
|
|
343
|
+
continue
|
|
344
|
+
# Root markers first; nested components keep their own evidence, not their parent's.
|
|
345
|
+
for name in reversed(sorted(dirs, key=lambda n: (n not in conventional, n))):
|
|
346
|
+
child_relative=(directory/name).relative_to(root).as_posix()
|
|
347
|
+
child_reason = reason or self.scope.folder_reason(child_relative, name, families)
|
|
348
|
+
if authored(child_relative) and child_reason!='vcs_metadata':child_reason=None
|
|
349
|
+
if child_reason=='wordpress_deployed_package' and self.include_installed_plugins and name=='plugins':child_reason=None
|
|
350
|
+
if child_reason in ARCHIVE_REASONS and self.include_archives:
|
|
351
|
+
child_reason = None
|
|
352
|
+
if name in OS_SKIP or (child_reason and not (child_reason!='vcs_metadata' and authored_descendant(child_relative))):
|
|
353
|
+
self.exclusions['dependency_generated_or_system'] += 1
|
|
354
|
+
continue
|
|
355
|
+
stack.append((directory / name, depth + 1, rules, parent, tuple(families),child_reason))
|
|
356
|
+
if cache.failures:
|
|
357
|
+
self.issue('unreadable_source_files', root)
|
|
358
|
+
finally:
|
|
359
|
+
cache.close()
|
|
360
|
+
|
|
361
|
+
def classify(self, row):
|
|
362
|
+
root = Path(row['path'])
|
|
363
|
+
cache = self.scope.ReadCache(root)
|
|
364
|
+
collected = row.pop('files')
|
|
365
|
+
local_config=self.scope.ScopeConfig(self.scope_config)
|
|
366
|
+
local_config.control=getattr(self.scope_config,'control',None)
|
|
367
|
+
prefix=row['relative_path'].rstrip('/')+'/' if row['relative_path']!='.' else ''
|
|
368
|
+
for key in ('include','exclude','source_roots'):
|
|
369
|
+
local_config[key]=[p[len(prefix):] if prefix and p.startswith(prefix) else p for p in local_config.get(key,[])]
|
|
370
|
+
local_config['roles']={p[len(prefix):] if prefix and p.startswith(prefix) else p:v for p,v in local_config.get('roles',{}).items()}
|
|
371
|
+
local_config['projects']=[{**p,'root':p['root'][len(prefix):] or '.'} for p in local_config.get('projects',[]) if not prefix or p['root'].startswith(prefix)]
|
|
372
|
+
selection = self.scope.Scope(root, respect_gitignore=self.respect_gitignore,config=local_config,
|
|
373
|
+
max_directories=self.limits['max_directories'],max_entries=self.limits['max_entries'],max_depth=self.limits['max_depth'],
|
|
374
|
+
max_read_bytes=max(0,self.limits['max_bytes']-self.stats['bytes_requested'])).build()
|
|
375
|
+
allowed = set(selection.files)
|
|
376
|
+
collected = sorted([p for p in collected if p in allowed],key=lambda p:selection.record(p)['priority'])
|
|
377
|
+
row['scope'] = selection.report()
|
|
378
|
+
from .facets import facets
|
|
379
|
+
row['facets'] = facets(selection,owner='root',max_files=32,max_bytes=max(0,min(262144,self.limits['max_bytes']-self.stats['bytes_requested'])))
|
|
380
|
+
self.stats['bytes_requested']+=selection.cache.bytes_read
|
|
381
|
+
if selection.cache.failures.get('read_budget'):self.issue('content_read_truncated',root)
|
|
382
|
+
if not selection.selection_complete():self.issue('classification_scope_partial',root)
|
|
383
|
+
selection.cache.close()
|
|
384
|
+
example_parts = {'examples', 'example', 'samples', 'sample', 'fixtures', '__fixtures__', 'templates', 'demo_projects'}
|
|
385
|
+
files = [p for p in collected if self.scope.file_role(p.relative_to(root).as_posix())
|
|
386
|
+
not in {'tests', 'fixture', 'docs'} and not (set(p.relative_to(root).parts[:-1]) & example_parts)]
|
|
387
|
+
row['classification_excluded_paths'] = len(collected) - len(files)
|
|
388
|
+
# Conventional app source before tooling/tests. Read at most 32 x 8 KB per project.
|
|
389
|
+
ordered = sorted(files, key=lambda p: (self.scope.file_role(p.relative_to(root).as_posix())
|
|
390
|
+
not in {'production', 'source', 'unknown'}, len(p.relative_to(root).parts), p.as_posix()))
|
|
391
|
+
names = {p.name.lower(): p for p in files}
|
|
392
|
+
paths = '\n'.join(p.relative_to(root).as_posix().lower() for p in files)
|
|
393
|
+
suffixes = Counter(self.scope.LANGUAGES[p.suffix.lower()] for p in files if p.suffix.lower() in self.scope.LANGUAGES)
|
|
394
|
+
row['languages_sampled'] = dict(suffixes.most_common())
|
|
395
|
+
try:
|
|
396
|
+
root_names = [p.name for p in files if p.parent == root]
|
|
397
|
+
if any(n in {'package.json', 'pyproject.toml', 'Cargo.toml', 'pnpm-workspace.yaml',
|
|
398
|
+
'settings.gradle', 'settings.gradle.kts'} or n.endswith('.sln') for n in root_names):
|
|
399
|
+
node = self.projects.node_for(root, '.', root_names,
|
|
400
|
+
lambda p, limit: self.read(cache, p, limit), root=True)
|
|
401
|
+
row['workspace_patterns'] = node['workspace_patterns']
|
|
402
|
+
row['metadata_issues'] = node['unknowns']
|
|
403
|
+
samples = [p for p in ordered if p.suffix.lower() in self.scope.LANGUAGES
|
|
404
|
+
and self.scope.file_role(p.relative_to(root).as_posix()) not in {'tests', 'fixture', 'docs'}][:32]
|
|
405
|
+
row['source_files_sampled'] = len(samples)
|
|
406
|
+
views = []
|
|
407
|
+
for p in samples:
|
|
408
|
+
try:
|
|
409
|
+
views.append(code_view(self.read(cache, p, 8192),
|
|
410
|
+
self.engine['adapters'].language(p.name) or self.scope.LANGUAGES[p.suffix.lower()], self.engine).lower())
|
|
411
|
+
except (ValueError, RecursionError):
|
|
412
|
+
row['sampling_limited'] = True
|
|
413
|
+
snippets = '\n'.join(views)
|
|
414
|
+
candidates = []
|
|
415
|
+
for ident, profile in self.registry.profiles.items():
|
|
416
|
+
score, evidence = 0, []
|
|
417
|
+
for rule in profile['detection']:
|
|
418
|
+
kind, value = rule['kind'], rule['value'].lower()
|
|
419
|
+
matched = False
|
|
420
|
+
if kind == 'file': matched = value in names
|
|
421
|
+
elif kind == 'extension': matched = any(p.suffix.lower() == value for p in files)
|
|
422
|
+
elif kind == 'path': matched = value in paths
|
|
423
|
+
elif kind == 'text': matched = value in snippets
|
|
424
|
+
elif kind == 'json_key':
|
|
425
|
+
name, _, key = rule['value'].partition(':')
|
|
426
|
+
if name.lower() in names:
|
|
427
|
+
try:
|
|
428
|
+
doc = json.loads(self.read(cache, names[name.lower()], 65536))
|
|
429
|
+
matched = isinstance(doc, dict) and key in doc
|
|
430
|
+
except (ValueError, RecursionError): pass
|
|
431
|
+
if matched:
|
|
432
|
+
score += rule['weight']
|
|
433
|
+
evidence.append({'kind': kind, 'signal': rule['value'], 'weight': rule['weight']})
|
|
434
|
+
if score:
|
|
435
|
+
candidates.append({'profile': ident, 'name': profile['name'], 'score': score, 'evidence': evidence})
|
|
436
|
+
# Structured dependencies improve on keyword-only inference (and don't run npm scripts).
|
|
437
|
+
if 'package.json' in names:
|
|
438
|
+
try:
|
|
439
|
+
package = json.loads(self.read(cache, names['package.json'], 131072))
|
|
440
|
+
if not isinstance(package, dict): package = {}
|
|
441
|
+
deps = set()
|
|
442
|
+
for key in ('dependencies', 'devDependencies', 'peerDependencies'):
|
|
443
|
+
if isinstance(package.get(key), dict): deps.update(package[key])
|
|
444
|
+
for profile, selected, weight in [('tauri_electron', {'electron', '@tauri-apps/api', '@tauri-apps/cli'}, 100),
|
|
445
|
+
('saas_or_web_app', {'next', 'react', 'vue', 'svelte', 'nuxt', '@angular/core'}, 65),
|
|
446
|
+
('api_backend', {'express', 'fastify', '@nestjs/core'}, 80),
|
|
447
|
+
('ai_pipeline', {'openai', 'langchain', '@langchain/core', 'ollama'}, 50),
|
|
448
|
+
('mcp_agent_server', {'@modelcontextprotocol/sdk'}, 80)]:
|
|
449
|
+
if deps & selected:
|
|
450
|
+
target = next((c for c in candidates if c['profile'] == profile), None)
|
|
451
|
+
if target is None:
|
|
452
|
+
target = {'profile': profile, 'name': self.registry.profiles[profile]['name'], 'score': 0, 'evidence': []}
|
|
453
|
+
candidates.append(target)
|
|
454
|
+
target['score'] += weight
|
|
455
|
+
target['evidence'].append({'kind': 'dependency_names', 'signal': sorted(deps & selected), 'weight': weight})
|
|
456
|
+
except (ValueError, RecursionError):
|
|
457
|
+
self.issue('unparseable_package_manifest', root)
|
|
458
|
+
if row.pop('plugin_header'):
|
|
459
|
+
candidates.append({'profile': 'wordpress_plugin', 'name': 'WordPress Plugin', 'score': 130,
|
|
460
|
+
'evidence': [{'kind': 'header', 'signal': 'Plugin Name header in plugin directory', 'weight': 130}]})
|
|
461
|
+
if 'wordpress' in row['ecosystems']:
|
|
462
|
+
candidates.append({'profile': 'wordpress_site', 'name': 'WordPress Installation', 'score': 200,
|
|
463
|
+
'evidence': [{'kind': 'root_layout', 'signal': 'wp-config.php and WordPress directories', 'weight': 200}]})
|
|
464
|
+
for p in files:
|
|
465
|
+
if p.suffix.lower() not in {'.csproj', '.fsproj', '.vbproj'}:
|
|
466
|
+
continue
|
|
467
|
+
manifest = self.read(cache, p, 131072)
|
|
468
|
+
if re.search(r'<(?:UseWPF|UseWindowsForms)>\s*true\s*</', manifest, re.I):
|
|
469
|
+
candidates.append({'profile': 'desktop_utility', 'name': self.registry.profiles['desktop_utility']['name'],
|
|
470
|
+
'score': 110, 'evidence': [{'kind': 'build_metadata', 'signal': 'WPF/Windows Forms enabled', 'weight': 110}]})
|
|
471
|
+
if re.search(r'\bSdk\s*=\s*["\']Microsoft\.NET\.Sdk\.Web["\']', manifest):
|
|
472
|
+
candidates.append({'profile': 'api_backend', 'name': self.registry.profiles['api_backend']['name'],
|
|
473
|
+
'score': 90, 'evidence': [{'kind': 'build_metadata', 'signal': 'ASP.NET Web SDK', 'weight': 90}]})
|
|
474
|
+
merged = {}
|
|
475
|
+
for candidate in candidates:
|
|
476
|
+
previous = merged.get(candidate['profile'])
|
|
477
|
+
if previous:
|
|
478
|
+
previous['score'] += candidate['score']
|
|
479
|
+
previous['evidence'].extend(candidate['evidence'])
|
|
480
|
+
else:
|
|
481
|
+
merged[candidate['profile']] = candidate
|
|
482
|
+
candidates = sorted(merged.values(), key=lambda c: (-c['score'], c['profile']))
|
|
483
|
+
for c in candidates:
|
|
484
|
+
c['probability'] = round(min(.95, c['score'] / (c['score'] + 25)), 4)
|
|
485
|
+
generic_only = bool(candidates) and all(e['kind'] == 'file' and e['signal'] == 'package.json'
|
|
486
|
+
for c in candidates for e in c['evidence'])
|
|
487
|
+
best = candidates[0]['probability'] if candidates else 0
|
|
488
|
+
ratio = candidates[1]['score'] / candidates[0]['score'] if len(candidates) > 1 else 0
|
|
489
|
+
unknown = round(min(.95, max(1 - best, .35 * ratio ** 2, .75 if generic_only else .05)), 4)
|
|
490
|
+
unknown_candidate = {'profile': 'unknown_or_general_app', 'name': 'Unclassified / General',
|
|
491
|
+
'probability': unknown, 'score': 0, 'evidence': []}
|
|
492
|
+
row['type'] = unknown_candidate if unknown >= best else candidates[0]
|
|
493
|
+
row['alternatives'] = [c for c in candidates if c is not row['type']][:4]
|
|
494
|
+
row['unknown_probability'] = unknown
|
|
495
|
+
row['confidence'] = ('low' if row['type']['profile'] == 'unknown_or_general_app' else
|
|
496
|
+
'high' if row['type']['probability'] >= .85 and not row['sampling_limited'] else
|
|
497
|
+
'medium' if row['type']['probability'] >= .65 else 'low')
|
|
498
|
+
if any(c['profile'] == 'android' for c in candidates):
|
|
499
|
+
row['ecosystems'] = sorted(set(row['ecosystems']) | {'android'})
|
|
500
|
+
row['classification_read_failures'] = sum(cache.failures.values())
|
|
501
|
+
if cache.failures:
|
|
502
|
+
self.issue('unreadable_classification_files', root)
|
|
503
|
+
finally:
|
|
504
|
+
cache.close()
|
|
505
|
+
|
|
506
|
+
def run(self, roots):
|
|
507
|
+
for root in roots:
|
|
508
|
+
self.walk(root)
|
|
509
|
+
for row in self.rows:
|
|
510
|
+
try:self.classify(row)
|
|
511
|
+
except (OSError,ValueError):
|
|
512
|
+
self.issue('classification_scope_error',row['path'])
|
|
513
|
+
row.pop('files',None);row.pop('plugin_header',None)
|
|
514
|
+
row.update(type={'profile':'unknown_or_general_app','name':'Unclassified / General','probability':1.0,'score':0,'evidence':[]},alternatives=[],unknown_probability=1.0,confidence='low',sampling_limited=True,languages_sampled={})
|
|
515
|
+
by_path = {r['path']: r for r in self.rows}
|
|
516
|
+
for row in self.rows:
|
|
517
|
+
parent = by_path.get(row['parent_project'])
|
|
518
|
+
if parent:
|
|
519
|
+
relative = Path(row['path']).relative_to(parent['path']).as_posix()
|
|
520
|
+
if any(fnmatch.fnmatchcase(relative, p) for p in parent['workspace_patterns']):
|
|
521
|
+
row['relationship'] = 'workspace_member'
|
|
522
|
+
for row in self.rows:
|
|
523
|
+
children = [r for r in self.rows if r['parent_project'] == row['path']]
|
|
524
|
+
row['component_types'] = [{'path': r['path'], 'profile': r['type']['profile'],
|
|
525
|
+
'relationship': r['relationship']} for r in children]
|
|
526
|
+
# Literal Gradle inclusion + an Android app member is stronger than a generic Gradle marker.
|
|
527
|
+
members = [r for r in children if r['relationship'] == 'workspace_member' and r['type']['profile'] == 'android']
|
|
528
|
+
if row['type']['profile'] == 'unknown_or_general_app' and 'jvm_gradle' in row['ecosystems'] and members:
|
|
529
|
+
probability = min(.7368, max(r['type']['probability'] for r in members))
|
|
530
|
+
row['type'] = {'profile': 'android', 'name': self.registry.profiles['android']['name'],
|
|
531
|
+
'score': 70, 'probability': probability,
|
|
532
|
+
'evidence': [{'kind': 'declared_workspace_member', 'signal': r['path'], 'weight': 70} for r in members]}
|
|
533
|
+
row['unknown_probability'] = round(1 - probability, 4)
|
|
534
|
+
row['confidence'] = 'medium'
|
|
535
|
+
row['ecosystems'] = sorted(set(row['ecosystems']) | {'android'})
|
|
536
|
+
files = [p for p in (self.engine['source'] / 'aqaf').rglob('*')
|
|
537
|
+
if p.is_file() and p.suffix in {'.py', '.json'} and '__pycache__' not in p.parts]
|
|
538
|
+
digest = hashlib.sha256()
|
|
539
|
+
for p in sorted(files):
|
|
540
|
+
digest.update(p.relative_to(self.engine['source']).as_posix().encode() + b'\0' + p.read_bytes())
|
|
541
|
+
return {'schema_version': 1, 'tool_version': TOOL_VERSION,
|
|
542
|
+
'status': 'partial' if self.stats['issues'] else 'complete_within_policy',
|
|
543
|
+
'probability_kind': 'uncalibrated_heuristic_estimate',
|
|
544
|
+
'probability_method': 'Independent profile scores/(scores+25), capped at 0.95; generic package.json alone stays unclassified. Alternatives can overlap and do not sum to one.',
|
|
545
|
+
'root_probability_method': 'Manifest plus direct source: 0.97; manifest or plugin header: 0.90; Git marker: 0.80; source layout only: 0.55. These are estimates, not measured frequencies.',
|
|
546
|
+
'knowledge': {'aqaf_source': str(self.engine['source']), 'aqaf_version': self.engine['version'],
|
|
547
|
+
'runtime_sha256': digest.hexdigest(), 'profile_count': len(self.registry.profiles),
|
|
548
|
+
'tool_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
|
549
|
+
'prior_predictions_used_as_training_labels': False},
|
|
550
|
+
'policy': {'respect_gitignore': self.respect_gitignore, 'follow_symlinks': False,
|
|
551
|
+
'execute_target_code': False, 'register_projects': False,
|
|
552
|
+
'include_archives': self.include_archives,
|
|
553
|
+
'include_installed_plugins': self.include_installed_plugins, 'exclude': list(self.exclude)},
|
|
554
|
+
'limits': self.limits, 'statistics': dict(self.stats), 'excluded': dict(self.exclusions),
|
|
555
|
+
'issues': self.issues, 'project_count': len(self.rows),
|
|
556
|
+
'projects': sorted(self.rows, key=lambda r: r['path']),
|
|
557
|
+
'limitations': ['Probabilities are heuristic estimates, not measured accuracy or certification.',
|
|
558
|
+
'Nested roots own their evidence; a parent is not classified from dependency/child source.',
|
|
559
|
+
'Source sampling is bounded. Unvisited/ignored/generated/dependency trees are not certified empty.',
|
|
560
|
+
'Lexical masks reduce comment/literal matches; aliases and type binding remain unresolved. Dependencies do not prove runtime use.',
|
|
561
|
+
'Prior scan classifications are context only, not independently verified labels.']}
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def render_table(doc):
|
|
565
|
+
lines = [f"Found {doc['project_count']} project folders ({doc['status']}).",
|
|
566
|
+
'Percentages are heuristic estimates, not calibrated probabilities.', '',
|
|
567
|
+
'Root % Type % Project type Ecosystem / language Folder']
|
|
568
|
+
for r in doc['projects']:
|
|
569
|
+
stack = ','.join(r['ecosystems'] or list(r['languages_sampled'])[:2]) or 'unknown'
|
|
570
|
+
path = json.dumps(r['path'], ensure_ascii=False)[1:-1]
|
|
571
|
+
lines.append(f"{r['root_probability']:6.0%} {r['type']['probability']:6.0%} {r['type']['name'][:35]:35} {stack[:23]:23} {path}")
|
|
572
|
+
if doc['issues']:
|
|
573
|
+
lines += ['', f"Partial coverage: {doc['statistics']['issues']} issues/limits; use --format json for details."]
|
|
574
|
+
return '\n'.join(lines) + '\n'
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def discover(roots, **options):
|
|
579
|
+
"""Passive bounded discovery; probabilities are explicitly uncalibrated."""
|
|
580
|
+
return Finder(load_engine(), **options).run(roots)
|