codemetrics-ai 0.5.0 → 0.6.0-preview.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -1
- package/dist/analyzer.d.ts +1 -0
- package/dist/analyzer.js +8 -4
- package/dist/cli.js +7 -3
- package/dist/dependency-evidence.d.ts +5 -0
- package/dist/dependency-evidence.js +30 -0
- package/dist/dependency-inventory.d.ts +67 -0
- package/dist/dependency-inventory.js +203 -0
- package/dist/dependency-metadata.d.ts +65 -0
- package/dist/dependency-metadata.js +125 -0
- package/dist/dependency-pnpm.d.ts +4 -0
- package/dist/dependency-pnpm.js +237 -0
- package/dist/dependency-spec.d.ts +13 -0
- package/dist/dependency-spec.js +30 -0
- package/dist/evidence-reader.js +3 -1
- package/dist/evidence-tools.js +4 -0
- package/dist/function-scoring.d.ts +2 -1
- package/dist/function-scoring.js +13 -24
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/python/ASYNC_ADAPTERS.md +38 -0
- package/dist/python/ASYNC_CONTEXT.md +58 -0
- package/dist/python/ASYNC_SCORING.md +174 -0
- package/dist/python/ASYNC_USAGE.md +62 -0
- package/dist/python/BEHAVIOR_REVIEW.md +84 -0
- package/dist/python/COVERAGE.md +96 -0
- package/dist/python/DECOMPOSITION.md +46 -0
- package/dist/python/DUPLICATION.md +36 -0
- package/dist/python/ERROR_HANDLING.md +45 -0
- package/dist/python/ERROR_HANDLING_CONTEXT.md +61 -0
- package/dist/python/ERROR_HANDLING_OPERATIONS.md +54 -0
- package/dist/python/ERROR_HANDLING_TRANSFERS.md +64 -0
- package/dist/python/METHOD_COMPLEXITY.md +46 -0
- package/dist/python/README.md +162 -0
- package/dist/python/analysis.py +54 -0
- package/dist/python/async_adapters.py +243 -0
- package/dist/python/async_bindings.py +180 -0
- package/dist/python/async_context.py +143 -0
- package/dist/python/async_helper_paths.py +186 -0
- package/dist/python/async_provenance.py +129 -0
- package/dist/python/async_scoring.py +203 -0
- package/dist/python/async_usage.py +244 -0
- package/dist/python/coverage_collect.py +121 -0
- package/dist/python/coverage_provenance.py +77 -0
- package/dist/python/coverage_review.py +273 -0
- package/dist/python/coverage_worker.py +46 -0
- package/dist/python/decomposition.py +199 -0
- package/dist/python/duplication.py +177 -0
- package/dist/python/error_handling.py +249 -0
- package/dist/python/error_handling_context.py +294 -0
- package/dist/python/error_handling_operations.py +228 -0
- package/dist/python/error_handling_transfers.py +211 -0
- package/dist/python/review_cli.py +36 -0
- package/dist/python/source_facts.py +293 -0
- package/dist/python/source_selection.py +157 -0
- package/dist/python-analyzer.d.ts +21 -0
- package/dist/python-analyzer.js +123 -0
- package/dist/python-async-adapters.d.ts +67 -0
- package/dist/python-async-adapters.js +197 -0
- package/dist/python-async-context.d.ts +73 -0
- package/dist/python-async-context.js +181 -0
- package/dist/python-async-scoring.d.ts +89 -0
- package/dist/python-async-scoring.js +249 -0
- package/dist/python-async-usage.d.ts +56 -0
- package/dist/python-async-usage.js +143 -0
- package/dist/python-cli.d.ts +2 -0
- package/dist/python-cli.js +53 -0
- package/dist/python-collect-cli.d.ts +2 -0
- package/dist/python-collect-cli.js +28 -0
- package/dist/python-complexity.d.ts +18 -0
- package/dist/python-complexity.js +36 -0
- package/dist/python-coverage.d.ts +63 -0
- package/dist/python-coverage.js +173 -0
- package/dist/python-decomposition.d.ts +56 -0
- package/dist/python-decomposition.js +102 -0
- package/dist/python-duplication.d.ts +65 -0
- package/dist/python-duplication.js +89 -0
- package/dist/python-error-context.d.ts +84 -0
- package/dist/python-error-context.js +165 -0
- package/dist/python-error-handling.d.ts +56 -0
- package/dist/python-error-handling.js +120 -0
- package/dist/python-error-operations.d.ts +44 -0
- package/dist/python-error-operations.js +155 -0
- package/dist/python-error-transfers.d.ts +38 -0
- package/dist/python-error-transfers.js +125 -0
- package/dist/python-evidence.d.ts +23 -0
- package/dist/python-evidence.js +255 -0
- package/dist/score-arithmetic.d.ts +1 -12
- package/dist/score-arithmetic.js +1 -40
- package/dist/shared-source-facts/arithmetic.d.mts +9 -0
- package/dist/shared-source-facts/arithmetic.mjs +32 -0
- package/dist/shared-source-facts/contract.d.mts +35 -0
- package/dist/shared-source-facts/contract.mjs +95 -0
- package/dist/shared-source-facts/population.d.mts +12 -0
- package/dist/shared-source-facts/population.mjs +41 -0
- package/dist/source-facts.d.ts +4 -0
- package/dist/source-facts.js +16 -0
- package/package.json +15 -6
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
"""Bounded exception context and syntactic patterns; no score or target execution."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import ast
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import sys
|
|
10
|
+
import tokenize
|
|
11
|
+
|
|
12
|
+
import error_handling as handlers
|
|
13
|
+
from duplication import digest, read_verified
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
POLICY = "python/exception-context-patterns-v1"
|
|
17
|
+
MAX_STATEMENTS = 32
|
|
18
|
+
MAX_FOLLOWING = 3
|
|
19
|
+
MAX_DOCUMENTATION_CHARS = 2000
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def fingerprint(value):
|
|
23
|
+
return digest(json.dumps(value, sort_keys=True, separators=(",", ":")))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def literal_kind(value):
|
|
27
|
+
if value is None:
|
|
28
|
+
return "none"
|
|
29
|
+
if isinstance(value, ast.Constant) and value.value is not Ellipsis:
|
|
30
|
+
return "none" if value.value is None else type(value.value).__name__
|
|
31
|
+
if isinstance(value, ast.UnaryOp) and isinstance(value.op, (ast.USub, ast.UAdd)) and \
|
|
32
|
+
isinstance(value.operand, ast.Constant) and type(value.operand.value) in (int, float, complex):
|
|
33
|
+
return type(value.operand.value).__name__
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def literal_return(node):
|
|
38
|
+
return literal_kind(node.value) if isinstance(node, ast.Return) else None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def literal_assignment(node):
|
|
42
|
+
if isinstance(node, ast.Assign) and all(isinstance(item, ast.Name) for item in node.targets) and literal_kind(node.value):
|
|
43
|
+
return {"targets": [item.id for item in node.targets], "valueKind": literal_kind(node.value)}
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def statement(node):
|
|
48
|
+
return {"kind": type(node).__name__, "line": node.lineno, "endLine": node.end_lineno,
|
|
49
|
+
"literalReturnKind": literal_return(node), "literalAssignment": literal_assignment(node)}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def suite(nodes, maximum=MAX_STATEMENTS):
|
|
53
|
+
return {"statementCount": len(nodes), "statements": [statement(node) for node in nodes[:maximum]],
|
|
54
|
+
"truncated": len(nodes) > maximum}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def import_names(node):
|
|
58
|
+
if isinstance(node, ast.Name):
|
|
59
|
+
return [node.id] if node.id in ("ImportError", "ModuleNotFoundError") else []
|
|
60
|
+
if isinstance(node, ast.Tuple):
|
|
61
|
+
parts = [import_names(item) for item in node.elts]
|
|
62
|
+
return [name for part in parts for name in part] if parts and all(parts) else []
|
|
63
|
+
return []
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def bound_names(tree):
|
|
67
|
+
"""Whole-file syntactic veto, deliberately conservative across unrelated scopes."""
|
|
68
|
+
names = set()
|
|
69
|
+
dynamic = False
|
|
70
|
+
for node in ast.walk(tree):
|
|
71
|
+
if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
|
|
72
|
+
names.add(node.id)
|
|
73
|
+
elif isinstance(node, ast.arg):
|
|
74
|
+
names.add(node.arg)
|
|
75
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
76
|
+
names.add(node.name)
|
|
77
|
+
elif isinstance(node, ast.ExceptHandler) and node.name:
|
|
78
|
+
names.add(node.name)
|
|
79
|
+
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
80
|
+
names.update(item.asname or item.name.split(".")[0] for item in node.names)
|
|
81
|
+
dynamic |= any(item.name == "*" for item in node.names)
|
|
82
|
+
elif isinstance(node, (ast.MatchAs, ast.MatchStar)) and node.name:
|
|
83
|
+
names.add(node.name)
|
|
84
|
+
elif isinstance(node, ast.MatchMapping) and node.rest:
|
|
85
|
+
names.add(node.rest)
|
|
86
|
+
elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in ("exec", "eval", "globals", "locals"):
|
|
87
|
+
dynamic = True
|
|
88
|
+
return names, dynamic
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class ContextVisitor(handlers.HandlerVisitor):
|
|
92
|
+
def __init__(self, source, record, owners, tree):
|
|
93
|
+
super().__init__(source, record, owners)
|
|
94
|
+
self.contexts, self.parents, self.suites = [], {}, {}
|
|
95
|
+
self.bindings, self.dynamic_bindings = bound_names(tree)
|
|
96
|
+
for parent in ast.walk(tree):
|
|
97
|
+
for field, value in ast.iter_fields(parent):
|
|
98
|
+
children = value if isinstance(value, list) else [value]
|
|
99
|
+
for index, child in enumerate(children):
|
|
100
|
+
if isinstance(child, ast.AST):
|
|
101
|
+
self.parents[child] = parent
|
|
102
|
+
if isinstance(value, list) and isinstance(child, ast.stmt):
|
|
103
|
+
self.suites[child] = (field, index, value)
|
|
104
|
+
|
|
105
|
+
def ancestors(self, node):
|
|
106
|
+
result = []
|
|
107
|
+
while node in self.parents:
|
|
108
|
+
node = self.parents[node]
|
|
109
|
+
result.append(node)
|
|
110
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
111
|
+
break
|
|
112
|
+
return result
|
|
113
|
+
|
|
114
|
+
def surrounding_documentation(self, attempt, ancestors):
|
|
115
|
+
comments = []
|
|
116
|
+
for index in range(attempt.lineno - 2, -1, -1):
|
|
117
|
+
line = self.lines[index]
|
|
118
|
+
if not line.strip().startswith("#") or len(line) - len(line.lstrip()) != attempt.col_offset:
|
|
119
|
+
break
|
|
120
|
+
comments.append((index + 1, line.lstrip()[1:].strip()))
|
|
121
|
+
comments.reverse()
|
|
122
|
+
result = []
|
|
123
|
+
for line, text in comments:
|
|
124
|
+
if text and not handlers.DIRECTIVE.match(text):
|
|
125
|
+
result.append({"kind": "preceding-try-comment", "line": line, "text": text})
|
|
126
|
+
owner = next((item for item in ancestors if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))), None)
|
|
127
|
+
if owner:
|
|
128
|
+
text = ast.get_docstring(owner, clean=False)
|
|
129
|
+
if text:
|
|
130
|
+
result.append({"kind": "owner-docstring", "line": owner.body[0].lineno, "text": text})
|
|
131
|
+
remaining, bounded = MAX_DOCUMENTATION_CHARS, []
|
|
132
|
+
for item in result:
|
|
133
|
+
if remaining <= 0:
|
|
134
|
+
break
|
|
135
|
+
text = item["text"][:remaining]
|
|
136
|
+
bounded.append({**item, "text": text, "truncated": len(text) < len(item["text"])})
|
|
137
|
+
remaining -= len(text)
|
|
138
|
+
return {"items": bounded, "truncated": len(bounded) < len(result) or any(item["truncated"] for item in bounded)}
|
|
139
|
+
|
|
140
|
+
def classify(self, node, attempt, following, row, ancestors):
|
|
141
|
+
inert = all(handlers.inert(item) for item in node.body)
|
|
142
|
+
# Documentation is observable even when surrounding execution is unresolved.
|
|
143
|
+
if inert and row["documentation"]:
|
|
144
|
+
return {"status": "recognized-shape", "pattern": "locally-documented-no-op",
|
|
145
|
+
"evidenceLines": [item["line"] for item in row["documentation"]], "unresolvedReasons": []}
|
|
146
|
+
blockers = []
|
|
147
|
+
if row["exceptionGroup"]:
|
|
148
|
+
blockers.append("exception-group-semantics")
|
|
149
|
+
if row["enclosingFinallyLines"]:
|
|
150
|
+
blockers.append("enclosing-finally")
|
|
151
|
+
if any(isinstance(item, (ast.With, ast.AsyncWith)) for item in ancestors):
|
|
152
|
+
blockers.append("enclosing-context-manager")
|
|
153
|
+
if any(isinstance(item, ast.ExceptHandler) for item in ancestors):
|
|
154
|
+
blockers.append("nested-handler")
|
|
155
|
+
if blockers:
|
|
156
|
+
return {"status": "unresolved", "pattern": None, "evidenceLines": [node.lineno], "unresolvedReasons": blockers}
|
|
157
|
+
pattern, evidence = None, [node.lineno]
|
|
158
|
+
if len(node.body) == 1 and literal_return(node.body[0]):
|
|
159
|
+
pattern, evidence = "direct-literal-fallback-return", evidence + [node.body[0].lineno]
|
|
160
|
+
elif len(node.body) == 1 and isinstance(node.body[0], ast.Raise) and node.body[0].exc is None:
|
|
161
|
+
pattern, evidence = "direct-bare-rethrow", evidence + [node.body[0].lineno]
|
|
162
|
+
elif len(attempt.body) == 1 and isinstance(attempt.body[0], (ast.Import, ast.ImportFrom)) and import_names(node.type):
|
|
163
|
+
imported = {item.asname or item.name.split(".")[0] for item in attempt.body[0].names}
|
|
164
|
+
default = literal_assignment(node.body[0]) if len(node.body) == 1 else None
|
|
165
|
+
if self.dynamic_bindings or any(name in self.bindings for name in import_names(node.type)):
|
|
166
|
+
blockers.append("import-exception-binding-unresolved")
|
|
167
|
+
elif inert:
|
|
168
|
+
pattern, evidence = "guarded-import-shape", [attempt.body[0].lineno, node.lineno]
|
|
169
|
+
elif default and set(default["targets"]).issubset(imported):
|
|
170
|
+
pattern, evidence = "guarded-import-literal-default", [attempt.body[0].lineno, node.lineno, node.body[0].lineno]
|
|
171
|
+
elif inert and following and literal_return(following[0]):
|
|
172
|
+
pattern, evidence = "no-op-then-literal-return", evidence + [following[0].lineno]
|
|
173
|
+
if pattern:
|
|
174
|
+
return {"status": "recognized-shape", "pattern": pattern, "evidenceLines": sorted(set(evidence)), "unresolvedReasons": []}
|
|
175
|
+
return {"status": "unresolved", "pattern": None, "evidenceLines": evidence,
|
|
176
|
+
"unresolvedReasons": blockers or ["no-supported-local-shape"]}
|
|
177
|
+
|
|
178
|
+
def handler(self, node, group):
|
|
179
|
+
offset = len(self.rows)
|
|
180
|
+
super().handler(node, group)
|
|
181
|
+
row = self.rows[offset]
|
|
182
|
+
attempt = self.parents[node]
|
|
183
|
+
field, index, siblings = self.suites[attempt]
|
|
184
|
+
following = siblings[index + 1:]
|
|
185
|
+
ancestors = self.ancestors(attempt)
|
|
186
|
+
self.contexts.append({"handlerId": row["id"], "file": row["file"], "line": row["line"],
|
|
187
|
+
"sourceSha256": row["sourceSha256"], "population": row["population"], "owner": row["owner"],
|
|
188
|
+
"tryStatement": {"line": attempt.lineno, "endLine": attempt.end_lineno, "kind": type(attempt).__name__,
|
|
189
|
+
"body": suite(attempt.body), "else": suite(attempt.orelse), "finally": suite(attempt.finalbody),
|
|
190
|
+
"handlerLines": [item.lineno for item in attempt.handlers]},
|
|
191
|
+
"handlerBody": suite(node.body),
|
|
192
|
+
"followingInSameSuite": {"suite": field, **suite(following, MAX_FOLLOWING)},
|
|
193
|
+
"enclosingConstructs": [{"kind": type(item).__name__, "line": item.lineno} for item in ancestors if hasattr(item, "lineno")],
|
|
194
|
+
"surroundingDocumentation": self.surrounding_documentation(attempt, ancestors),
|
|
195
|
+
"classification": self.classify(node, attempt, following, row, ancestors), "disposition": "review-only"})
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def summarize(rows):
|
|
199
|
+
def counts(items):
|
|
200
|
+
return {"handlers": len(items), "recognizedShapes": sum(item["classification"]["status"] == "recognized-shape" for item in items),
|
|
201
|
+
"unresolved": sum(item["classification"]["status"] == "unresolved" for item in items),
|
|
202
|
+
"byPattern": dict(sorted(Counter(item["classification"]["pattern"] for item in items if item["classification"]["pattern"]).items())),
|
|
203
|
+
"unresolvedReasons": dict(sorted(Counter(reason for item in items for reason in item["classification"]["unresolvedReasons"]).items()))}
|
|
204
|
+
return {**counts(rows), "populations": {kind: counts([row for row in rows if row["population"] == kind]) for kind in sorted({row["population"] for row in rows})}}
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def review(target, facts, base):
|
|
208
|
+
configuration = {"policy": POLICY, "handlerPolicy": handlers.POLICY, "maxStatementsPerSuite": MAX_STATEMENTS,
|
|
209
|
+
"maxFollowingStatements": MAX_FOLLOWING, "maxDocumentationCharacters": MAX_DOCUMENTATION_CHARS,
|
|
210
|
+
"selectionFingerprint": facts["selection"]["configurationFingerprint"], "parserVersion": facts["producer"]["parserVersion"]}
|
|
211
|
+
report = {"schemaVersion": 1, "artifactKind": "experimental-python-exception-context-review",
|
|
212
|
+
"producer": {"name": "codemetrics-ai/python-exception-context", "version": "0.1.0-experimental"},
|
|
213
|
+
"generatedAtUtc": datetime.now(timezone.utc).isoformat(), "runId": base["runId"], "auditId": base["auditId"],
|
|
214
|
+
"policy": POLICY, "configuration": configuration, "configurationFingerprint": fingerprint(configuration),
|
|
215
|
+
"handlerReview": base, "handlerRowsSha256": fingerprint(base["handlers"]),
|
|
216
|
+
"status": base["status"], "diagnostics": list(base["diagnostics"]), "contexts": [], "summary": None,
|
|
217
|
+
"scope": "Bounded lexical context and syntactic patterns, not resolved behavior, optionality, handling success, defects or scores. Following statements are lexical siblings, not guaranteed execution. Surrounding prose does not automatically explain a particular handler."}
|
|
218
|
+
if facts["status"] != "complete" or base["status"] != "complete":
|
|
219
|
+
report["status"] = "incomplete"
|
|
220
|
+
report["diagnostics"] = list(base["diagnostics"] or facts["diagnostics"])
|
|
221
|
+
return report
|
|
222
|
+
target = Path(target).absolute()
|
|
223
|
+
root = target if target.is_dir() else target.parent
|
|
224
|
+
try:
|
|
225
|
+
if base["selection"] != facts["selection"] or base["sourceFactsProducer"] != facts["producer"] or base["sourceFactsPolicy"] != facts["measurementPolicy"] or base["policy"] != handlers.POLICY or base["subject"] != {"root": str(root), "entryPoint": str(target)}:
|
|
226
|
+
raise ValueError("Handler/source-facts provenance mismatch.")
|
|
227
|
+
owners = {item["id"]: item for item in facts["functions"]}
|
|
228
|
+
if len(owners) != len(facts["functions"]):
|
|
229
|
+
raise ValueError("Repeated selected function identity.")
|
|
230
|
+
rows, regenerated, seen = [], [], set()
|
|
231
|
+
for record in facts["selection"]["files"]:
|
|
232
|
+
if record["disposition"] != "analyzed":
|
|
233
|
+
continue
|
|
234
|
+
source = read_verified(root, record)
|
|
235
|
+
tree = ast.parse(source, filename=record["file"])
|
|
236
|
+
for node in ast.walk(tree):
|
|
237
|
+
if isinstance(node, handlers.source_facts.OWNERS):
|
|
238
|
+
key = digest(f"{record['file']}|{node.lineno}|{node.col_offset}|{type(node).__name__}")
|
|
239
|
+
if key in owners:
|
|
240
|
+
seen.add(key)
|
|
241
|
+
visitor = ContextVisitor(source, record, owners, tree)
|
|
242
|
+
visitor.visit(tree)
|
|
243
|
+
rows.extend(visitor.contexts)
|
|
244
|
+
regenerated.extend(visitor.rows)
|
|
245
|
+
if len(rows) > handlers.MAX_HANDLERS:
|
|
246
|
+
raise ValueError("Exception context population exceeds the handler budget.")
|
|
247
|
+
regenerated.sort(key=lambda row: (row["file"], row["line"], row["id"]))
|
|
248
|
+
if seen != set(owners) or regenerated != base["handlers"] or handlers.summarize(regenerated) != base["summary"]:
|
|
249
|
+
raise ValueError("Handler observations changed during context analysis.")
|
|
250
|
+
rows.sort(key=lambda row: (row["file"], row["line"], row["handlerId"]))
|
|
251
|
+
report["contexts"], report["summary"] = rows, summarize(rows)
|
|
252
|
+
except (OSError, SyntaxError, UnicodeError, ValueError, RecursionError, tokenize.TokenError) as error:
|
|
253
|
+
report["status"] = "incomplete"
|
|
254
|
+
report["diagnostics"].append({"code": "exception-context-analysis", "message": str(error)})
|
|
255
|
+
return report
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def analyze(target, *, run_id=None, audit_id=None, **selection):
|
|
259
|
+
facts = handlers.source_facts.analyze(target, **selection)
|
|
260
|
+
base = handlers.review(target, facts, run_id=run_id, audit_id=audit_id)
|
|
261
|
+
return review(target, facts, base)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def main():
|
|
265
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
266
|
+
parser.add_argument("target", help="Python source file or directory")
|
|
267
|
+
parser.add_argument("--output", type=Path, help="New .json file with an existing parent")
|
|
268
|
+
parser.add_argument("--run-id")
|
|
269
|
+
parser.add_argument("--audit-id")
|
|
270
|
+
for name in ("include", "exclude", "test-path", "production-path"):
|
|
271
|
+
parser.add_argument(f"--{name}", action="append", default=[])
|
|
272
|
+
for name in ("include-tests", "include-generated", "include-support"):
|
|
273
|
+
parser.add_argument(f"--{name}", action="store_true")
|
|
274
|
+
args = parser.parse_args()
|
|
275
|
+
try:
|
|
276
|
+
report = analyze(args.target, run_id=args.run_id, audit_id=args.audit_id, include=args.include, exclude=args.exclude,
|
|
277
|
+
test_paths=args.test_path, production_paths=args.production_path, include_tests=args.include_tests,
|
|
278
|
+
include_generated=args.include_generated, include_support=args.include_support)
|
|
279
|
+
payload = json.dumps(report, indent=2, ensure_ascii=True) + "\n"
|
|
280
|
+
if args.output:
|
|
281
|
+
if args.output.suffix != ".json":
|
|
282
|
+
raise ValueError("Output must be a new .json file.")
|
|
283
|
+
with args.output.open("x", encoding="utf-8") as handle:
|
|
284
|
+
handle.write(payload)
|
|
285
|
+
else:
|
|
286
|
+
sys.stdout.write(payload)
|
|
287
|
+
return 0 if report["status"] == "complete" else 2
|
|
288
|
+
except (OSError, ValueError) as error:
|
|
289
|
+
print(f"Cannot produce exception context review: {error}", file=sys.stderr)
|
|
290
|
+
return 2
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
if __name__ == "__main__":
|
|
294
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""Owned handler call/return syntax and unresolved-shape inventory; no score."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import ast
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import sys
|
|
10
|
+
import tokenize
|
|
11
|
+
|
|
12
|
+
import error_handling_transfers as transfers
|
|
13
|
+
from duplication import digest, read_verified
|
|
14
|
+
|
|
15
|
+
context = transfers.context
|
|
16
|
+
POLICY = "python/exception-call-return-v1"
|
|
17
|
+
MAX_OPERATIONS = 100000
|
|
18
|
+
MAX_ANCESTORS = 32
|
|
19
|
+
LOGGING_NAMES = frozenset({"debug", "info", "warning", "warn", "error", "exception", "critical", "fatal", "log"})
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def owned_operations(handler, parents, file):
|
|
23
|
+
calls, returns = [], []
|
|
24
|
+
|
|
25
|
+
def location(node):
|
|
26
|
+
ancestors, parent = [], parents[node]
|
|
27
|
+
while parent is not handler:
|
|
28
|
+
if isinstance(parent, (ast.stmt, ast.expr)):
|
|
29
|
+
ancestors.append({"kind": type(parent).__name__, "line": parent.lineno})
|
|
30
|
+
parent = parents[parent]
|
|
31
|
+
return {"id": digest(f"{file}|{node.lineno}|{node.col_offset}|{node.end_lineno}|{node.end_col_offset}|{type(node).__name__}"),
|
|
32
|
+
"line": node.lineno, "column": node.col_offset, "endLine": node.end_lineno, "endColumn": node.end_col_offset,
|
|
33
|
+
"ancestors": ancestors[:MAX_ANCESTORS], "ancestorsTruncated": len(ancestors) > MAX_ANCESTORS}
|
|
34
|
+
|
|
35
|
+
def visit(node):
|
|
36
|
+
if isinstance(node, context.handlers.source_facts.OWNERS + (ast.ClassDef, ast.ExceptHandler)):
|
|
37
|
+
return
|
|
38
|
+
if isinstance(node, ast.Call):
|
|
39
|
+
parent = parents[node]
|
|
40
|
+
awaited = isinstance(parent, ast.Await)
|
|
41
|
+
statement = parents[parent] if awaited else parent
|
|
42
|
+
symbol = transfers.symbolic_path(node.func)
|
|
43
|
+
# Lexical hint only: neither imports nor receiver identity are resolved.
|
|
44
|
+
terminal = symbol.split(".")[-1] if symbol else None
|
|
45
|
+
calls.append({**location(node), "callee": transfers.expression(node.func, handler.name),
|
|
46
|
+
"loggingNameSyntax": terminal if terminal in LOGGING_NAMES else None,
|
|
47
|
+
"directStatement": isinstance(statement, ast.Expr) and statement in handler.body,
|
|
48
|
+
"immediatelyAwaited": awaited, "positionalArguments": len(node.args),
|
|
49
|
+
"keywordArguments": len(node.keywords),
|
|
50
|
+
"starredArguments": sum(isinstance(item, ast.Starred) for item in node.args),
|
|
51
|
+
"keywordUnpacks": sum(item.arg is None for item in node.keywords),
|
|
52
|
+
"containsNamedExpression": any(isinstance(item, ast.NamedExpr) for item in ast.walk(node))})
|
|
53
|
+
elif isinstance(node, ast.Return):
|
|
54
|
+
returns.append({**location(node), "direct": node in handler.body,
|
|
55
|
+
"expression": transfers.expression(node.value, handler.name),
|
|
56
|
+
"awaitedExpression": transfers.expression(node.value.value, handler.name) if isinstance(node.value, ast.Await) else None,
|
|
57
|
+
"containsNamedExpression": any(isinstance(item, ast.NamedExpr) for item in ast.walk(node))})
|
|
58
|
+
if len(calls) + len(returns) > MAX_OPERATIONS:
|
|
59
|
+
raise ValueError("Call/return observation budget exceeded.")
|
|
60
|
+
for child in ast.iter_child_nodes(node):
|
|
61
|
+
visit(child)
|
|
62
|
+
|
|
63
|
+
for statement in handler.body:
|
|
64
|
+
visit(statement)
|
|
65
|
+
order = lambda row: (row["line"], row["column"])
|
|
66
|
+
return sorted(calls, key=order), sorted(returns, key=order)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def classify(handler, prior):
|
|
70
|
+
# Never relax earlier blockers or rewrite any previously recognized pattern.
|
|
71
|
+
if prior["status"] == "recognized-shape" or prior["unresolvedReasons"] != ["no-supported-local-shape"]:
|
|
72
|
+
return dict(prior)
|
|
73
|
+
if any(isinstance(item, ast.NamedExpr) for statement in handler.body for item in ast.walk(statement)):
|
|
74
|
+
return dict(prior)
|
|
75
|
+
body, pattern = handler.body, None
|
|
76
|
+
if len(body) == 1 and isinstance(body[0], ast.Return):
|
|
77
|
+
value = body[0].value
|
|
78
|
+
if isinstance(value, (ast.Name, ast.Attribute, ast.Call)) and transfers.expression(value, handler.name)["symbol"]:
|
|
79
|
+
pattern = "direct-symbol-or-call-return"
|
|
80
|
+
else:
|
|
81
|
+
rethrow = len(body) > 1 and isinstance(body[-1], ast.Raise) and body[-1].exc is None
|
|
82
|
+
prefix = body[:-1] if rethrow else body
|
|
83
|
+
if prefix and all(isinstance(item, ast.Expr) and isinstance(item.value, ast.Call) and
|
|
84
|
+
transfers.symbolic_path(item.value.func) is not None for item in prefix):
|
|
85
|
+
pattern = "direct-calls-then-bare-raise" if rethrow else "direct-call-statement-sequence"
|
|
86
|
+
if not pattern:
|
|
87
|
+
return dict(prior)
|
|
88
|
+
return {"status": "recognized-shape", "pattern": pattern,
|
|
89
|
+
"evidenceLines": sorted({handler.lineno, *(statement.lineno for statement in body)}), "unresolvedReasons": []}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class OperationVisitor(transfers.TransferVisitor):
|
|
93
|
+
def __init__(self, source, record, owners, tree):
|
|
94
|
+
super().__init__(source, record, owners, tree)
|
|
95
|
+
self.operations = []
|
|
96
|
+
|
|
97
|
+
def handler(self, node, group):
|
|
98
|
+
super().handler(node, group)
|
|
99
|
+
original = self.transfers[-1]
|
|
100
|
+
calls, returns = owned_operations(node, self.parents, original["file"])
|
|
101
|
+
self.operations.append({key: original[key] for key in ("handlerId", "file", "line", "sourceSha256", "population", "owner")} |
|
|
102
|
+
{"calls": calls, "returns": returns, "suite": context.suite(node.body),
|
|
103
|
+
"originalClassification": original["classification"],
|
|
104
|
+
"classification": classify(node, original["classification"]), "disposition": "review-only"})
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def inventory(rows):
|
|
108
|
+
"""Inventory the input's unresolved population before any new classification."""
|
|
109
|
+
items = [row for row in rows if row["originalClassification"]["status"] == "unresolved"]
|
|
110
|
+
shapes = Counter((tuple(value["kind"] for value in row["suite"]["statements"]), row["suite"]["truncated"]) for row in items)
|
|
111
|
+
return {"handlers": len(items), "singleStatement": sum(row["suite"]["statementCount"] == 1 for row in items),
|
|
112
|
+
"multiStatement": sum(row["suite"]["statementCount"] > 1 for row in items),
|
|
113
|
+
"withCalls": sum(bool(row["calls"]) for row in items), "withReturns": sum(bool(row["returns"]) for row in items),
|
|
114
|
+
"withLoggingNameSyntax": sum(any(call["loggingNameSyntax"] for call in row["calls"]) for row in items),
|
|
115
|
+
"byReason": dict(sorted(Counter(reason for row in items for reason in row["originalClassification"]["unresolvedReasons"]).items())),
|
|
116
|
+
"suiteShapes": [{"kinds": list(shape), "truncated": truncated, "handlers": count}
|
|
117
|
+
for (shape, truncated), count in sorted(shapes.items(), key=lambda pair: (-pair[1], pair[0]))]}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def summarize(rows):
|
|
121
|
+
def counts(items):
|
|
122
|
+
calls = [call for row in items for call in row["calls"]]
|
|
123
|
+
returns = [value for row in items for value in row["returns"]]
|
|
124
|
+
return {**{key: value for key, value in context.summarize(items).items() if key != "populations"},
|
|
125
|
+
"callExpressions": len(calls), "returnStatements": len(returns),
|
|
126
|
+
"handlersWithCalls": sum(bool(row["calls"]) for row in items),
|
|
127
|
+
"handlersWithReturns": sum(bool(row["returns"]) for row in items),
|
|
128
|
+
"loggingNameSyntaxCalls": sum(call["loggingNameSyntax"] is not None for call in calls),
|
|
129
|
+
"immediatelyAwaitedCalls": sum(call["immediatelyAwaited"] for call in calls),
|
|
130
|
+
"byReturnExpression": dict(sorted(Counter(value["expression"]["kind"] for value in returns).items())),
|
|
131
|
+
"newlyRecognizedShapes": sum(row["originalClassification"]["status"] == "unresolved" and row["classification"]["status"] == "recognized-shape" for row in items),
|
|
132
|
+
"inputUnresolvedInventory": inventory(items)}
|
|
133
|
+
return {**counts(rows), "populations": {kind: counts([row for row in rows if row["population"] == kind]) for kind in sorted({row["population"] for row in rows})}}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def review(target, facts, base):
|
|
137
|
+
configuration = {"policy": POLICY, "transferPolicy": transfers.POLICY, "transferConfigurationFingerprint": base["configurationFingerprint"],
|
|
138
|
+
"maxOperations": MAX_OPERATIONS, "maxAncestors": MAX_ANCESTORS, "maxSuiteStatements": context.MAX_STATEMENTS,
|
|
139
|
+
"maxSymbolCharacters": transfers.MAX_PATH_CHARACTERS, "maxSymbolAttributes": transfers.MAX_SYMBOL_ATTRIBUTES,
|
|
140
|
+
"loggingNames": sorted(LOGGING_NAMES)}
|
|
141
|
+
report = {"schemaVersion": 1, "artifactKind": "experimental-python-exception-operation-review",
|
|
142
|
+
"producer": {"name": "codemetrics-ai/python-exception-operations", "version": "0.2.0-experimental"},
|
|
143
|
+
"generatedAtUtc": datetime.now(timezone.utc).isoformat(), "runId": base["runId"], "auditId": base["auditId"],
|
|
144
|
+
"policy": POLICY, "configuration": configuration, "configurationFingerprint": context.fingerprint(configuration),
|
|
145
|
+
"transferReview": base, "transferRowsSha256": context.fingerprint(base["handlers"]),
|
|
146
|
+
"status": base["status"], "diagnostics": list(base["diagnostics"]), "handlers": [], "summary": None,
|
|
147
|
+
"scope": "Owned call/return syntax and input unresolved-shape inventory; not resolved logging, delegation, recovery, runtime paths, defects, or scores."}
|
|
148
|
+
if facts["status"] != "complete" or base["status"] != "complete":
|
|
149
|
+
report["status"] = "incomplete"
|
|
150
|
+
report["diagnostics"] = list(base["diagnostics"] or facts["diagnostics"])
|
|
151
|
+
return report
|
|
152
|
+
target = Path(target).absolute()
|
|
153
|
+
root = target if target.is_dir() else target.parent
|
|
154
|
+
try:
|
|
155
|
+
regenerated = transfers.review(target, facts, base["contextReview"])
|
|
156
|
+
stable = lambda value: {key: item for key, item in value.items() if key != "generatedAtUtc"}
|
|
157
|
+
if regenerated["status"] != "complete" or stable(regenerated) != stable(base):
|
|
158
|
+
raise ValueError("Raise/chaining evidence changed or has inconsistent provenance.")
|
|
159
|
+
owners = {item["id"]: item for item in facts["functions"]}
|
|
160
|
+
rows, prior, contexts, handlers, count = [], [], [], [], 0
|
|
161
|
+
for record in facts["selection"]["files"]:
|
|
162
|
+
if record["disposition"] != "analyzed":
|
|
163
|
+
continue
|
|
164
|
+
source = read_verified(root, record)
|
|
165
|
+
tree = ast.parse(source, filename=record["file"])
|
|
166
|
+
visitor = OperationVisitor(source, record, owners, tree)
|
|
167
|
+
visitor.visit(tree)
|
|
168
|
+
rows.extend(visitor.operations)
|
|
169
|
+
prior.extend(visitor.transfers)
|
|
170
|
+
contexts.extend(visitor.contexts)
|
|
171
|
+
handlers.extend(visitor.rows)
|
|
172
|
+
count += sum(len(row["calls"]) + len(row["returns"]) for row in visitor.operations)
|
|
173
|
+
if count > MAX_OPERATIONS or len(rows) > context.handlers.MAX_HANDLERS:
|
|
174
|
+
raise ValueError("Call/return observation population exceeds its budget.")
|
|
175
|
+
order = lambda row: (row["file"], row["line"], row["handlerId"])
|
|
176
|
+
rows.sort(key=order); prior.sort(key=order); contexts.sort(key=order)
|
|
177
|
+
handlers.sort(key=lambda row: (row["file"], row["line"], row["id"]))
|
|
178
|
+
if prior != base["handlers"] or contexts != base["contextReview"]["contexts"] or handlers != base["contextReview"]["handlerReview"]["handlers"]:
|
|
179
|
+
raise ValueError("Earlier observations changed during call/return analysis.")
|
|
180
|
+
for row, original in zip(rows, handlers):
|
|
181
|
+
projected = [{"line": value["line"], "hasValue": value["expression"]["kind"] != "absent"} for value in row["returns"]]
|
|
182
|
+
if len(row["calls"]) != original["bodyObservations"]["calls"] or projected != original["bodyObservations"]["returns"]:
|
|
183
|
+
raise ValueError("Owned calls/returns disagree with original observations.")
|
|
184
|
+
report["handlers"], report["summary"] = rows, summarize(rows)
|
|
185
|
+
except (OSError, SyntaxError, UnicodeError, ValueError, RecursionError, tokenize.TokenError) as error:
|
|
186
|
+
report["status"] = "incomplete"
|
|
187
|
+
report["diagnostics"].append({"code": "exception-operation-analysis", "message": str(error)})
|
|
188
|
+
return report
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def analyze(target, *, run_id=None, audit_id=None, **selection):
|
|
192
|
+
facts = context.handlers.source_facts.analyze(target, **selection)
|
|
193
|
+
handlers = context.handlers.review(target, facts, run_id=run_id, audit_id=audit_id)
|
|
194
|
+
base_context = context.review(target, facts, handlers)
|
|
195
|
+
return review(target, facts, transfers.review(target, facts, base_context))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def main():
|
|
199
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
200
|
+
parser.add_argument("target", help="Python source file or directory")
|
|
201
|
+
parser.add_argument("--output", type=Path, help="New .json file with an existing parent")
|
|
202
|
+
parser.add_argument("--run-id")
|
|
203
|
+
parser.add_argument("--audit-id")
|
|
204
|
+
for name in ("include", "exclude", "test-path", "production-path"):
|
|
205
|
+
parser.add_argument(f"--{name}", action="append", default=[])
|
|
206
|
+
for name in ("include-tests", "include-generated", "include-support"):
|
|
207
|
+
parser.add_argument(f"--{name}", action="store_true")
|
|
208
|
+
args = parser.parse_args()
|
|
209
|
+
try:
|
|
210
|
+
report = analyze(args.target, run_id=args.run_id, audit_id=args.audit_id, include=args.include, exclude=args.exclude,
|
|
211
|
+
test_paths=args.test_path, production_paths=args.production_path, include_tests=args.include_tests,
|
|
212
|
+
include_generated=args.include_generated, include_support=args.include_support)
|
|
213
|
+
payload = json.dumps(report, indent=2, ensure_ascii=True) + "\n"
|
|
214
|
+
if args.output:
|
|
215
|
+
if args.output.suffix != ".json":
|
|
216
|
+
raise ValueError("Output must be a new .json file.")
|
|
217
|
+
with args.output.open("x", encoding="utf-8") as handle:
|
|
218
|
+
handle.write(payload)
|
|
219
|
+
else:
|
|
220
|
+
sys.stdout.write(payload)
|
|
221
|
+
return 0 if report["status"] == "complete" else 2
|
|
222
|
+
except (OSError, ValueError) as error:
|
|
223
|
+
print(f"Cannot produce exception call/return review: {error}", file=sys.stderr)
|
|
224
|
+
return 2
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
if __name__ == "__main__":
|
|
228
|
+
raise SystemExit(main())
|