adl-toolkit 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTALL_TOOLKIT.md +163 -0
- package/README.md +269 -2
- package/adl_drift/__init__.py +6 -0
- package/adl_drift/cli/__init__.py +1 -0
- package/adl_drift/cli/commands.py +993 -0
- package/adl_drift/cli/main.py +744 -0
- package/adl_drift/cortex/__init__.py +1 -0
- package/adl_drift/cortex/embeddings.py +96 -0
- package/adl_drift/cortex/memory.py +1749 -0
- package/adl_drift/cortex/patterns.py +89 -0
- package/adl_drift/cortex/telemetry.py +192 -0
- package/adl_drift/detectors/__init__.py +1 -0
- package/adl_drift/detectors/async_patterns.py +115 -0
- package/adl_drift/detectors/base.py +121 -0
- package/adl_drift/detectors/config.py +122 -0
- package/adl_drift/detectors/data_access.py +128 -0
- package/adl_drift/detectors/decorators.py +122 -0
- package/adl_drift/detectors/dynamic_patterns.py +260 -0
- package/adl_drift/detectors/errors.py +134 -0
- package/adl_drift/detectors/imports.py +132 -0
- package/adl_drift/detectors/logging_patterns.py +108 -0
- package/adl_drift/detectors/routes.py +257 -0
- package/adl_drift/detectors/security.py +236 -0
- package/adl_drift/detectors/taint.py +406 -0
- package/adl_drift/detectors/tests.py +95 -0
- package/adl_drift/detectors/type_conventions.py +81 -0
- package/adl_drift/graph/__init__.py +1 -0
- package/adl_drift/graph/builder.py +2670 -0
- package/adl_drift/graph/cfg.py +461 -0
- package/adl_drift/graph/cluster.py +40 -0
- package/adl_drift/graph/edges.py +86 -0
- package/adl_drift/graph/node.py +90 -0
- package/adl_drift/harness/__init__.py +10 -0
- package/adl_drift/harness/setup.py +832 -0
- package/adl_drift/harness/skills/drift-antigravity.md +19 -0
- package/adl_drift/harness/skills/drift-generic.md +19 -0
- package/adl_drift/harness/skills/drift.md +25 -0
- package/adl_drift/harness/watch.py +84 -0
- package/adl_drift/lang/__init__.py +8 -0
- package/adl_drift/lang/base.py +464 -0
- package/adl_drift/lang/java/__init__.py +6 -0
- package/adl_drift/lang/java/backend.py +97 -0
- package/adl_drift/lang/java/cfg.py +1424 -0
- package/adl_drift/lang/java/config.py +707 -0
- package/adl_drift/lang/java/detectors/__init__.py +29 -0
- package/adl_drift/lang/java/detectors/additional.py +445 -0
- package/adl_drift/lang/java/detectors/base.py +154 -0
- package/adl_drift/lang/java/detectors/data_access.py +148 -0
- package/adl_drift/lang/java/detectors/imports.py +53 -0
- package/adl_drift/lang/java/detectors/routes.py +159 -0
- package/adl_drift/lang/java/detectors/security.py +386 -0
- package/adl_drift/lang/java/detectors/tests.py +74 -0
- package/adl_drift/lang/java/extractor.py +582 -0
- package/adl_drift/lang/java/ids.py +99 -0
- package/adl_drift/lang/java/parser.py +49 -0
- package/adl_drift/lang/java/resolution/__init__.py +1 -0
- package/adl_drift/lang/java/resolution/constants.py +1619 -0
- package/adl_drift/lang/java/resolution/constructors.py +278 -0
- package/adl_drift/lang/java/resolution/context.py +305 -0
- package/adl_drift/lang/java/resolution/dispatch.py +540 -0
- package/adl_drift/lang/java/resolution/functional.py +1318 -0
- package/adl_drift/lang/java/resolution/generated.py +95 -0
- package/adl_drift/lang/java/resolution/model.py +153 -0
- package/adl_drift/lang/java/resolution/orchestration.py +866 -0
- package/adl_drift/lang/java/resolution/overloads.py +1275 -0
- package/adl_drift/lang/java/resolution/scopes.py +1111 -0
- package/adl_drift/lang/java/resolution/types.py +2947 -0
- package/adl_drift/lang/java/resolver.py +1401 -0
- package/adl_drift/lang/java/scopes.py +69 -0
- package/adl_drift/lang/java/taint.py +500 -0
- package/adl_drift/lang/java/type_provenance.py +342 -0
- package/adl_drift/lang/python/__init__.py +8 -0
- package/adl_drift/lang/python/backend.py +94 -0
- package/adl_drift/lang/python/cfg.py +537 -0
- package/adl_drift/lang/python/extractor.py +454 -0
- package/adl_drift/lang/python/parser.py +41 -0
- package/adl_drift/lang/python/resolver.py +1789 -0
- package/adl_drift/lang/python/taint.py +365 -0
- package/adl_drift/lang/registry.py +117 -0
- package/adl_drift/parser/__init__.py +1 -0
- package/adl_drift/parser/cache.py +38 -0
- package/adl_drift/parser/core.py +33 -0
- package/adl_drift/parser/extractor.py +17 -0
- package/adl_drift/parser/literal_eval.py +326 -0
- package/adl_drift/scanner/__init__.py +1 -0
- package/adl_drift/scanner/discovery.py +338 -0
- package/adl_drift/scanner/incremental.py +261 -0
- package/adl_drift/scanner/parallel.py +224 -0
- package/adl_drift/server/__init__.py +1 -0
- package/adl_drift/server/handlers.py +2849 -0
- package/adl_drift/server/mcp_server.py +127 -0
- package/adl_drift/server/mcp_stdio.py +335 -0
- package/adl_drift/server/protocol.py +71 -0
- package/adl_drift/server/static/index.html +696 -0
- package/adl_drift/server/web.py +350 -0
- package/adl_drift/utils/__init__.py +1 -0
- package/adl_drift/utils/config.py +77 -0
- package/adl_drift/utils/fswalk.py +52 -0
- package/adl_drift/utils/types.py +345 -0
- package/bin/adl-drift.mjs +35 -0
- package/bin/adl-toolkit.mjs +488 -0
- package/package.json +29 -4
- package/pyproject.toml +49 -0
- package/scripts/install_toolkit.py +3584 -0
|
@@ -0,0 +1,461 @@
|
|
|
1
|
+
"""Intra-function Control Flow Graph + Reaching Definitions analysis (P2.18,
|
|
2
|
+
generalized to a per-language seam in P4.6 — see PROBLEMS.md P4.6).
|
|
3
|
+
|
|
4
|
+
Two sub-components, both language-agnostic:
|
|
5
|
+
A. CFGBuilderBase — shared block/def bookkeeping (block+def id allocation,
|
|
6
|
+
the `_link`/`_new_block`/`_new_def` primitives) that a per-language
|
|
7
|
+
subclass drives to walk a function CST into basic blocks. Python's is
|
|
8
|
+
`lang/python/cfg.py::PythonCFGBuilder`; Java's is
|
|
9
|
+
`lang/java/cfg.py::JavaCFGBuilder`. Neither this base class nor
|
|
10
|
+
`ReachingDefinitions` below ever inspects a tree-sitter node type —
|
|
11
|
+
that CST-coupled dispatch (which statement kinds branch/loop/terminate,
|
|
12
|
+
what counts as a literal or a taint source) lives entirely in the
|
|
13
|
+
subclass, driven by the `CFGConfig` a `LanguageBackend.cfg_config()`
|
|
14
|
+
supplies (mirrors how P4.4 split `ResolverConfig`'s declarative policy
|
|
15
|
+
from `CallResolver`'s CST-coupled mechanics).
|
|
16
|
+
B. ReachingDefinitions — runs the standard GEN/KILL fixpoint over the
|
|
17
|
+
CFGBlock graph any language's builder produces, and sweeps each block
|
|
18
|
+
to emit DefUseFact triples. Fully generic: it only ever touches
|
|
19
|
+
CFGBlock/CFGDef, never a CST node.
|
|
20
|
+
|
|
21
|
+
Facts are stored in SQLite (CortexMemory.store_def_use_facts) and queried by
|
|
22
|
+
SecurityDetector to promote/demote severity and by DynamicPatternDetector to
|
|
23
|
+
tighten the resolvable= signal for getattr/exec dynamic patterns (both
|
|
24
|
+
Python-only consumers).
|
|
25
|
+
|
|
26
|
+
Key design choices (unchanged from P2.18):
|
|
27
|
+
- MAX_DEF_USE_FACTS_PER_FUNCTION = 10_000 cap (spec requirement).
|
|
28
|
+
- Conservative handling of unrecognised statement types: all identifiers
|
|
29
|
+
are treated as uses (never missed), no defs emitted (may miss some defs
|
|
30
|
+
in unusual syntax — safe to overapproximate in both directions).
|
|
31
|
+
- Parameters are defs in the entry block with is_literal=False and
|
|
32
|
+
taint_source=None (a parameter *could* be tainted; callers decide) —
|
|
33
|
+
except where a language's own source table marks a parameter as always
|
|
34
|
+
tainted (Java's `@RequestParam` et al.; see `lang/java/cfg.py`).
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
from collections import deque
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from typing import Dict, List, Optional, Set, Tuple, Any
|
|
42
|
+
|
|
43
|
+
from tree_sitter import Node
|
|
44
|
+
|
|
45
|
+
from adl_drift.lang.base import CFGConfig
|
|
46
|
+
from adl_drift.utils.types import SourcePosition
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ─── tuning constants ─────────────────────────────────────────────────────────
|
|
50
|
+
|
|
51
|
+
MAX_DEF_USE_FACTS_PER_FUNCTION = 10_000
|
|
52
|
+
|
|
53
|
+
# ─── Python-only literal check, used by detectors/security.py's P2.18 eval/exec
|
|
54
|
+
# severity refinement. Deliberately NOT part of the generic CFGConfig-driven
|
|
55
|
+
# engine below — it is a standalone text/node-type check unrelated to CFG
|
|
56
|
+
# construction, kept here unchanged so that pre-existing import stays valid:
|
|
57
|
+
# `from adl_drift.graph.cfg import _is_literal as cfg_is_literal`. Java has
|
|
58
|
+
# no equivalent consumer (JavaSecurityDetector does not do def-use severity
|
|
59
|
+
# refinement), so this is not generalized.
|
|
60
|
+
|
|
61
|
+
_LITERAL_TYPES: frozenset = frozenset({
|
|
62
|
+
"integer", "float", "string", "concatenated_string",
|
|
63
|
+
"true", "false", "none",
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _is_literal(node: Node) -> bool:
|
|
68
|
+
"""True when node represents a compile-time constant (Python literals)."""
|
|
69
|
+
if node.type in _LITERAL_TYPES:
|
|
70
|
+
return True
|
|
71
|
+
if node.type == "unary_operator":
|
|
72
|
+
op = node.child_by_field_name("operator")
|
|
73
|
+
arg = node.child_by_field_name("argument")
|
|
74
|
+
if op and _text(op, b"-") == "-" and arg and arg.type in {"integer", "float"}:
|
|
75
|
+
return True
|
|
76
|
+
if node.type == "parenthesized_expression" and node.child_count >= 3:
|
|
77
|
+
inner = node.children[1]
|
|
78
|
+
return _is_literal(inner)
|
|
79
|
+
return False
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# ─── generic tree-sitter helpers (grammar-agnostic byte/position math) ───────
|
|
83
|
+
|
|
84
|
+
def _node_pos(node: Node) -> SourcePosition:
|
|
85
|
+
return SourcePosition(
|
|
86
|
+
line=node.start_point[0] + 1,
|
|
87
|
+
column=node.start_point[1],
|
|
88
|
+
end_line=node.end_point[0] + 1,
|
|
89
|
+
end_column=node.end_point[1],
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _text(node: Node, source: bytes) -> str:
|
|
94
|
+
try:
|
|
95
|
+
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
96
|
+
except Exception:
|
|
97
|
+
return ""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ─── public data types ────────────────────────────────────────────────────────
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class DefUseFact:
|
|
104
|
+
"""A reaching-definition relationship within one function body.
|
|
105
|
+
|
|
106
|
+
Interpretation:
|
|
107
|
+
``def_site`` is an assignment of ``binding_name``, and there exists a
|
|
108
|
+
control-flow path from ``def_site`` to ``use_site`` along which no other
|
|
109
|
+
assignment to ``binding_name`` occurs (standard reaching-definition).
|
|
110
|
+
|
|
111
|
+
``is_literal`` — the RHS at ``def_site`` is a compile-time constant.
|
|
112
|
+
``taint_source`` — the RHS is derived from a known user-input source.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
function_id: str
|
|
116
|
+
binding_name: str
|
|
117
|
+
def_site: SourcePosition
|
|
118
|
+
use_site: SourcePosition
|
|
119
|
+
is_literal: bool = False
|
|
120
|
+
taint_source: Optional[str] = None
|
|
121
|
+
|
|
122
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
123
|
+
return {
|
|
124
|
+
"function_id": self.function_id,
|
|
125
|
+
"binding_name": self.binding_name,
|
|
126
|
+
"def_site": self.def_site.to_dict(),
|
|
127
|
+
"use_site": self.use_site.to_dict(),
|
|
128
|
+
"is_literal": self.is_literal,
|
|
129
|
+
"taint_source": self.taint_source,
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def from_dict(cls, d: Dict[str, Any]) -> "DefUseFact":
|
|
134
|
+
return cls(
|
|
135
|
+
function_id=d["function_id"],
|
|
136
|
+
binding_name=d["binding_name"],
|
|
137
|
+
def_site=SourcePosition.from_dict(d["def_site"]),
|
|
138
|
+
use_site=SourcePosition.from_dict(d["use_site"]),
|
|
139
|
+
is_literal=d.get("is_literal", False),
|
|
140
|
+
taint_source=d.get("taint_source"),
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# ─── internal CFG types (shared by every language's builder) ─────────────────
|
|
145
|
+
|
|
146
|
+
@dataclass
|
|
147
|
+
class CFGDef:
|
|
148
|
+
idx: int # global index within this function
|
|
149
|
+
name: str
|
|
150
|
+
position: SourcePosition
|
|
151
|
+
is_literal: bool
|
|
152
|
+
taint_source: Optional[str]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@dataclass
|
|
156
|
+
class CFGUse:
|
|
157
|
+
name: str
|
|
158
|
+
position: SourcePosition
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@dataclass
|
|
162
|
+
class CFGBlock:
|
|
163
|
+
id: int
|
|
164
|
+
# events: interleaved CFGDef / CFGUse objects in textual order.
|
|
165
|
+
# Preserving order lets the sweep correctly pair uses with their
|
|
166
|
+
# preceding defs (a use only sees defs from earlier in the same block).
|
|
167
|
+
events: List[Any] = field(default_factory=list)
|
|
168
|
+
succs: List[int] = field(default_factory=list)
|
|
169
|
+
preds: List[int] = field(default_factory=list)
|
|
170
|
+
# Dataflow sets (def indices)
|
|
171
|
+
gen: Set[int] = field(default_factory=set)
|
|
172
|
+
kill: Set[int] = field(default_factory=set)
|
|
173
|
+
inn: Set[int] = field(default_factory=set)
|
|
174
|
+
out: Set[int] = field(default_factory=set)
|
|
175
|
+
|
|
176
|
+
# Convenience views (populated lazily by _compute_gen_kill)
|
|
177
|
+
@property
|
|
178
|
+
def defs(self) -> List[CFGDef]:
|
|
179
|
+
return [e for e in self.events if isinstance(e, CFGDef)]
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def uses(self) -> List[CFGUse]:
|
|
183
|
+
return [e for e in self.events if isinstance(e, CFGUse)]
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# ─── CFG builder base (shared bookkeeping; CST dispatch is per-language) ─────
|
|
187
|
+
|
|
188
|
+
class CFGBuilderBase:
|
|
189
|
+
"""Shared block/def bookkeeping for CFG construction (P4.6).
|
|
190
|
+
|
|
191
|
+
Owns nothing CST-coupled: block/def id allocation and the block-graph
|
|
192
|
+
edge primitives (`_new_block`/`_new_def`/`_link`). `build()` — walking a
|
|
193
|
+
function CST node into basic blocks — is implemented by a per-language
|
|
194
|
+
subclass driven by `self.config` (a `CFGConfig`): `lang/python/cfg.py::
|
|
195
|
+
PythonCFGBuilder` (the original P2.18 Python dispatch, ported unchanged
|
|
196
|
+
behavior) and `lang/java/cfg.py::JavaCFGBuilder`.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
def __init__(self, config: CFGConfig) -> None:
|
|
200
|
+
self.config = config
|
|
201
|
+
self._block_id: int = 0
|
|
202
|
+
self._def_id: int = 0
|
|
203
|
+
self._blocks: Dict[int, CFGBlock] = {}
|
|
204
|
+
self._all_defs: List[CFGDef] = []
|
|
205
|
+
|
|
206
|
+
def build(
|
|
207
|
+
self, func_node: Node, source: bytes
|
|
208
|
+
) -> Tuple[int, int, Dict[int, CFGBlock], List[CFGDef]]:
|
|
209
|
+
"""Return ``(entry_id, exit_id, blocks, all_defs)`` for `func_node`.
|
|
210
|
+
|
|
211
|
+
``entry_id`` is the entry basic block. ``exit_id`` is the synthetic
|
|
212
|
+
sink block (all returns/raises/throws connect here). ``all_defs`` is
|
|
213
|
+
every `CFGDef`, indexed by `CFGDef.idx`. Implemented per language.
|
|
214
|
+
"""
|
|
215
|
+
raise NotImplementedError
|
|
216
|
+
|
|
217
|
+
# ── shared bookkeeping ────────────────────────────────────────────────
|
|
218
|
+
|
|
219
|
+
def _reset(self) -> None:
|
|
220
|
+
self._block_id = 0
|
|
221
|
+
self._def_id = 0
|
|
222
|
+
self._blocks = {}
|
|
223
|
+
self._all_defs = []
|
|
224
|
+
|
|
225
|
+
def _new_block(self) -> CFGBlock:
|
|
226
|
+
b = CFGBlock(id=self._block_id)
|
|
227
|
+
self._block_id += 1
|
|
228
|
+
self._blocks[b.id] = b
|
|
229
|
+
return b
|
|
230
|
+
|
|
231
|
+
def _new_def(
|
|
232
|
+
self, name: str, pos: SourcePosition, is_lit: bool, taint: Optional[str]
|
|
233
|
+
) -> CFGDef:
|
|
234
|
+
d = CFGDef(idx=self._def_id, name=name, position=pos,
|
|
235
|
+
is_literal=is_lit, taint_source=taint)
|
|
236
|
+
self._def_id += 1
|
|
237
|
+
self._all_defs.append(d)
|
|
238
|
+
return d
|
|
239
|
+
|
|
240
|
+
def _link(self, src: CFGBlock, dst: CFGBlock) -> None:
|
|
241
|
+
if dst.id not in src.succs:
|
|
242
|
+
src.succs.append(dst.id)
|
|
243
|
+
if src.id not in dst.preds:
|
|
244
|
+
dst.preds.append(src.id)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# ─── reaching definitions (100% generic — never touches a CST node) ─────────
|
|
248
|
+
|
|
249
|
+
class ReachingDefinitions:
|
|
250
|
+
"""Runs the GEN/KILL fixpoint over a CFG and emits DefUseFact triples."""
|
|
251
|
+
|
|
252
|
+
def analyze(
|
|
253
|
+
self,
|
|
254
|
+
entry_id: int,
|
|
255
|
+
exit_id: int,
|
|
256
|
+
blocks: Dict[int, CFGBlock],
|
|
257
|
+
all_defs: List[CFGDef],
|
|
258
|
+
function_id: str,
|
|
259
|
+
) -> List[DefUseFact]:
|
|
260
|
+
"""Return all reaching-definition facts for the function.
|
|
261
|
+
|
|
262
|
+
Capped at MAX_DEF_USE_FACTS_PER_FUNCTION; functions that overflow are
|
|
263
|
+
tagged by returning an empty list (the overflow is reported via the
|
|
264
|
+
*overflow* flag on the exit block — callers can detect it via
|
|
265
|
+
``blocks[exit_id].gen``).
|
|
266
|
+
"""
|
|
267
|
+
if not blocks:
|
|
268
|
+
return []
|
|
269
|
+
|
|
270
|
+
# ── Step 1: build per-name def index ─────────────────────────────
|
|
271
|
+
# defs_by_name[name] = sorted list of def indices for that name
|
|
272
|
+
defs_by_name: Dict[str, List[int]] = {}
|
|
273
|
+
for d in all_defs:
|
|
274
|
+
defs_by_name.setdefault(d.name, []).append(d.idx)
|
|
275
|
+
|
|
276
|
+
# ── Step 2: compute GEN and KILL per block ────────────────────────
|
|
277
|
+
for b in blocks.values():
|
|
278
|
+
seen_names: Dict[str, int] = {} # name → last def idx seen in block
|
|
279
|
+
for d in b.defs:
|
|
280
|
+
seen_names[d.name] = d.idx
|
|
281
|
+
b.gen = set(seen_names.values())
|
|
282
|
+
|
|
283
|
+
# KILL[B] = all defs from *other* blocks for names defined in B
|
|
284
|
+
for name in seen_names:
|
|
285
|
+
for idx in defs_by_name.get(name, []):
|
|
286
|
+
if idx not in b.gen:
|
|
287
|
+
b.kill.add(idx)
|
|
288
|
+
|
|
289
|
+
# ── Step 3: fixpoint (worklist, reverse post-order approx) ───────
|
|
290
|
+
# OUT[B] = GEN[B] ∪ (IN[B] − KILL[B])
|
|
291
|
+
# IN[B] = ∪ OUT[P] for all predecessors P
|
|
292
|
+
|
|
293
|
+
# Initialise
|
|
294
|
+
for b in blocks.values():
|
|
295
|
+
b.inn = set()
|
|
296
|
+
b.out = set(b.gen)
|
|
297
|
+
|
|
298
|
+
worklist: deque = deque(blocks.keys())
|
|
299
|
+
seen_in_wl: Set[int] = set(blocks.keys())
|
|
300
|
+
|
|
301
|
+
while worklist:
|
|
302
|
+
bid = worklist.popleft()
|
|
303
|
+
seen_in_wl.discard(bid)
|
|
304
|
+
b = blocks[bid]
|
|
305
|
+
|
|
306
|
+
new_in: Set[int] = set()
|
|
307
|
+
for pred_id in b.preds:
|
|
308
|
+
new_in |= blocks[pred_id].out
|
|
309
|
+
|
|
310
|
+
new_out = b.gen | (new_in - b.kill)
|
|
311
|
+
|
|
312
|
+
if new_out != b.out or new_in != b.inn:
|
|
313
|
+
b.inn = new_in
|
|
314
|
+
b.out = new_out
|
|
315
|
+
for succ_id in b.succs:
|
|
316
|
+
if succ_id not in seen_in_wl:
|
|
317
|
+
worklist.append(succ_id)
|
|
318
|
+
seen_in_wl.add(succ_id)
|
|
319
|
+
|
|
320
|
+
# ── Step 4: sweep each block and emit DefUseFacts ─────────────────
|
|
321
|
+
# Walk the interleaved events list in textual order so that a def on
|
|
322
|
+
# line N is visible only to uses on line N+1+, not to earlier uses.
|
|
323
|
+
def_by_idx: Dict[int, CFGDef] = {d.idx: d for d in all_defs}
|
|
324
|
+
|
|
325
|
+
facts: List[DefUseFact] = []
|
|
326
|
+
overflow = False
|
|
327
|
+
|
|
328
|
+
for b in blocks.values():
|
|
329
|
+
reaching: Set[int] = set(b.inn)
|
|
330
|
+
|
|
331
|
+
for event in b.events:
|
|
332
|
+
if isinstance(event, CFGUse):
|
|
333
|
+
for idx in list(reaching):
|
|
334
|
+
d = def_by_idx.get(idx)
|
|
335
|
+
if d is None or d.name != event.name:
|
|
336
|
+
continue
|
|
337
|
+
facts.append(DefUseFact(
|
|
338
|
+
function_id=function_id,
|
|
339
|
+
binding_name=event.name,
|
|
340
|
+
def_site=d.position,
|
|
341
|
+
use_site=event.position,
|
|
342
|
+
is_literal=d.is_literal,
|
|
343
|
+
taint_source=d.taint_source,
|
|
344
|
+
))
|
|
345
|
+
if len(facts) >= MAX_DEF_USE_FACTS_PER_FUNCTION:
|
|
346
|
+
overflow = True
|
|
347
|
+
break
|
|
348
|
+
elif isinstance(event, CFGDef):
|
|
349
|
+
# Kill previous defs of same name, then add this one
|
|
350
|
+
for prev_idx in list(reaching):
|
|
351
|
+
prev = def_by_idx.get(prev_idx)
|
|
352
|
+
if prev and prev.name == event.name:
|
|
353
|
+
reaching.discard(prev_idx)
|
|
354
|
+
reaching.add(event.idx)
|
|
355
|
+
|
|
356
|
+
if overflow:
|
|
357
|
+
break
|
|
358
|
+
if overflow:
|
|
359
|
+
blocks[exit_id].gen.add(-1)
|
|
360
|
+
return facts
|
|
361
|
+
|
|
362
|
+
return facts
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
# ─── public entry points ──────────────────────────────────────────────────────
|
|
366
|
+
|
|
367
|
+
def _default_python_cfg_config() -> CFGConfig:
|
|
368
|
+
from adl_drift.lang.python.cfg import default_python_cfg_config
|
|
369
|
+
return default_python_cfg_config()
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def analyze_function(
|
|
373
|
+
func_node: Node,
|
|
374
|
+
source: bytes,
|
|
375
|
+
function_id: str,
|
|
376
|
+
cfg_config: Optional[CFGConfig] = None,
|
|
377
|
+
) -> List[DefUseFact]:
|
|
378
|
+
"""Build CFG + run reaching-definitions for one function. Returns facts.
|
|
379
|
+
|
|
380
|
+
``cfg_config`` defaults to Python's (`lang/python/cfg.py::
|
|
381
|
+
default_python_cfg_config`) when omitted — every pre-P4.6 call site
|
|
382
|
+
(which never passes one) is unaffected."""
|
|
383
|
+
if cfg_config is None:
|
|
384
|
+
cfg_config = _default_python_cfg_config()
|
|
385
|
+
if cfg_config.cfg_builder_factory is None:
|
|
386
|
+
raise ValueError(
|
|
387
|
+
"CFGConfig.cfg_builder_factory must be set "
|
|
388
|
+
"(see lang/python/cfg.py::default_python_cfg_config)."
|
|
389
|
+
)
|
|
390
|
+
builder = cfg_config.cfg_builder_factory(cfg_config)
|
|
391
|
+
entry_id, exit_id, blocks, all_defs = builder.build(func_node, source)
|
|
392
|
+
rd = ReachingDefinitions()
|
|
393
|
+
return rd.analyze(entry_id, exit_id, blocks, all_defs, function_id)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def analyze_file(
|
|
397
|
+
root_node: Node,
|
|
398
|
+
source: bytes,
|
|
399
|
+
module_name: str,
|
|
400
|
+
node_id_map: Dict[str, str],
|
|
401
|
+
cfg_config: Optional[CFGConfig] = None,
|
|
402
|
+
) -> List[DefUseFact]:
|
|
403
|
+
"""Analyze all top-level and class-method functions in a file.
|
|
404
|
+
|
|
405
|
+
``node_id_map`` maps ``(module.Class.func)`` → graph node id.
|
|
406
|
+
Used to set ``DefUseFact.function_id`` correctly.
|
|
407
|
+
Falls back to a simple qualified-name heuristic when the map doesn't
|
|
408
|
+
contain an entry.
|
|
409
|
+
|
|
410
|
+
Python-specific: the module-level walk (``_walk_for_functions``) looks
|
|
411
|
+
for `function_definition`/`class_definition` CST nodes, so this
|
|
412
|
+
convenience entry point only makes sense for Python sources today — the
|
|
413
|
+
CLI/server def-use pass (P4.10, not this issue's scope) only ever calls
|
|
414
|
+
it on Python files. Java callers should build their own file-level walk
|
|
415
|
+
(or call `analyze_function` directly per method, as
|
|
416
|
+
`lang/java/cfg.py`'s tests do) rather than reusing this function.
|
|
417
|
+
"""
|
|
418
|
+
facts: List[DefUseFact] = []
|
|
419
|
+
_walk_for_functions(root_node, source, module_name, [], node_id_map, facts, cfg_config)
|
|
420
|
+
return facts
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _walk_for_functions(
|
|
424
|
+
node: Node,
|
|
425
|
+
source: bytes,
|
|
426
|
+
module: str,
|
|
427
|
+
scope: List[str],
|
|
428
|
+
node_id_map: Dict[str, str],
|
|
429
|
+
out: List[DefUseFact],
|
|
430
|
+
cfg_config: Optional[CFGConfig],
|
|
431
|
+
) -> None:
|
|
432
|
+
"""DFS: find function_definition nodes and analyze them."""
|
|
433
|
+
t = node.type
|
|
434
|
+
|
|
435
|
+
if t in ("function_definition", "async_function_definition"):
|
|
436
|
+
name_node = node.child_by_field_name("name")
|
|
437
|
+
fname = _text(name_node, source) if name_node else "<anon>"
|
|
438
|
+
full_scope = scope + [fname]
|
|
439
|
+
fid_key = module + "." + ".".join(full_scope)
|
|
440
|
+
fid = node_id_map.get(fid_key, fid_key)
|
|
441
|
+
try:
|
|
442
|
+
facts = analyze_function(node, source, fid, cfg_config)
|
|
443
|
+
except Exception:
|
|
444
|
+
facts = []
|
|
445
|
+
out.extend(facts)
|
|
446
|
+
# Recurse into body for nested functions (closures)
|
|
447
|
+
body = node.child_by_field_name("body")
|
|
448
|
+
if body:
|
|
449
|
+
_walk_for_functions(body, source, module, full_scope, node_id_map, out, cfg_config)
|
|
450
|
+
return
|
|
451
|
+
|
|
452
|
+
if t in ("class_definition",):
|
|
453
|
+
name_node = node.child_by_field_name("name")
|
|
454
|
+
cname = _text(name_node, source) if name_node else "<anon_cls>"
|
|
455
|
+
body = node.child_by_field_name("body")
|
|
456
|
+
if body:
|
|
457
|
+
_walk_for_functions(body, source, module, scope + [cname], node_id_map, out, cfg_config)
|
|
458
|
+
return
|
|
459
|
+
|
|
460
|
+
for child in node.children:
|
|
461
|
+
_walk_for_functions(child, source, module, scope, node_id_map, out, cfg_config)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Community detection over the call graph.
|
|
2
|
+
|
|
3
|
+
Louvain runs on the directed graph (no `to_undirected()` cast). networkx's
|
|
4
|
+
`louvain_communities` accepts DiGraph natively and treats edges as directed,
|
|
5
|
+
so caller/callee semantics are preserved — a utility called by many should
|
|
6
|
+
land in a different community than its callers.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Dict
|
|
12
|
+
|
|
13
|
+
import networkx as nx
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def detect_communities(graph: nx.DiGraph, seed: int = 42) -> Dict[str, int]:
|
|
17
|
+
"""Run Louvain on `graph`; return {node_id: community_id}.
|
|
18
|
+
|
|
19
|
+
Deterministic via fixed `seed`. Returns empty dict for empty graphs.
|
|
20
|
+
Community ids are zero-based ints, dense.
|
|
21
|
+
"""
|
|
22
|
+
if graph.number_of_nodes() == 0:
|
|
23
|
+
return {}
|
|
24
|
+
if graph.is_multigraph():
|
|
25
|
+
# Louvain doesn't accept multigraphs — collapse parallel edges
|
|
26
|
+
# (direction preserved; edge multiplicity is irrelevant for clustering)
|
|
27
|
+
graph = nx.DiGraph(graph)
|
|
28
|
+
try:
|
|
29
|
+
communities = nx.algorithms.community.louvain_communities(
|
|
30
|
+
graph, seed=seed, weight=None
|
|
31
|
+
)
|
|
32
|
+
except Exception:
|
|
33
|
+
# Pathological graphs (e.g., disconnected singletons) -> fallback:
|
|
34
|
+
# each weakly-connected component gets its own community.
|
|
35
|
+
communities = list(nx.weakly_connected_components(graph))
|
|
36
|
+
out: Dict[str, int] = {}
|
|
37
|
+
for idx, members in enumerate(communities):
|
|
38
|
+
for nid in members:
|
|
39
|
+
out[nid] = idx
|
|
40
|
+
return out
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Edge definitions for call graph.
|
|
2
|
+
|
|
3
|
+
Represents calls, imports, inheritance, and other relationships.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from enum import Enum, auto
|
|
10
|
+
from typing import Optional, Dict, Any
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class EdgeType(Enum):
|
|
14
|
+
"""Types of edges in the graph."""
|
|
15
|
+
CALLS = auto() # function A calls function B
|
|
16
|
+
IMPORTS = auto() # module A imports module B
|
|
17
|
+
INHERITS = auto() # class A inherits from class B
|
|
18
|
+
IMPLEMENTS = auto() # class A implements interface B
|
|
19
|
+
CONTAINS = auto() # module/class A contains function B
|
|
20
|
+
DECORATED_BY = auto() # function A is decorated by decorator B
|
|
21
|
+
AWAITS = auto() # async function A awaits function B
|
|
22
|
+
REFERENCES = auto() # function A reads variable/constant B (P1.8)
|
|
23
|
+
TYPE_OF = auto() # function A has a parameter annotated with class B (P1.8)
|
|
24
|
+
RETURNS = auto() # function A's return annotation is class B (P1.8)
|
|
25
|
+
INSTANTIATES = auto() # function A constructs class B, i.e. B(...) (P2.8)
|
|
26
|
+
OVERRIDES = auto() # method A overrides ancestor method B (P2.8)
|
|
27
|
+
DYNAMIC_ATTR = auto() # caller reads a dynamic attr on a __getattr__ class (P2.14)
|
|
28
|
+
CALLABLE_REF = auto() # Java method/constructor reference (P4.13)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class GraphEdge:
|
|
33
|
+
"""An edge in the call graph."""
|
|
34
|
+
source: str # Source node ID
|
|
35
|
+
target: str # Target node ID
|
|
36
|
+
edge_type: EdgeType
|
|
37
|
+
file_path: Optional[str] = None
|
|
38
|
+
line: Optional[int] = None
|
|
39
|
+
# 0-indexed column of the call-site node itself (tree-sitter `start_point[1]`).
|
|
40
|
+
# Disambiguates two calls to the same simple name that resolve to different
|
|
41
|
+
# targets but land on the same source line, e.g. `good.process(q); bad.process("x");`
|
|
42
|
+
# — see `detectors/taint.py::TaintAnalyzer._resolve_callees` (F-03, PROBLEMS.md P4.6).
|
|
43
|
+
column: Optional[int] = None
|
|
44
|
+
# Exact CST byte range for call-site identity. Nested fluent invocations
|
|
45
|
+
# can share the same begin line/column, while their end byte remains
|
|
46
|
+
# distinct (`builder.step().step()`). Optional for structural/legacy
|
|
47
|
+
# edges and old persisted graphs.
|
|
48
|
+
start_byte: Optional[int] = None
|
|
49
|
+
end_byte: Optional[int] = None
|
|
50
|
+
|
|
51
|
+
# For call edges: arguments passed
|
|
52
|
+
call_args: Optional[str] = None
|
|
53
|
+
|
|
54
|
+
# Resolver confidence in [0.0, 1.0]. Structural edges (CONTAINS, INHERITS,
|
|
55
|
+
# IMPLEMENTS, DECORATED_BY, IMPORTS) are exact -> 1.0. CALLS / AWAITS are populated
|
|
56
|
+
# from CallGraphBuilder._resolve_target() strategy scores (0.3-0.95).
|
|
57
|
+
confidence: float = 1.0
|
|
58
|
+
strategy: Optional[str] = None
|
|
59
|
+
|
|
60
|
+
# Optional per-edge provenance. P4.12 (Java abstract-type dispatch fan-out)
|
|
61
|
+
# populates it with `strategy` (why the edge exists — e.g. "single_implementor",
|
|
62
|
+
# "interface_fanout", "spring_qualifier") and `declared_receiver_type` (the
|
|
63
|
+
# interface/abstract type the call was statically typed as), so impact/MCP can
|
|
64
|
+
# explain a fanned-out CALLS edge to a concrete implementor (deep-dive §8.4).
|
|
65
|
+
# Empty for every edge the other resolvers emit — additive, never required.
|
|
66
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
67
|
+
|
|
68
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
69
|
+
result = {
|
|
70
|
+
"source": self.source,
|
|
71
|
+
"target": self.target,
|
|
72
|
+
"type": self.edge_type.name,
|
|
73
|
+
"position": (
|
|
74
|
+
{"file": self.file_path, "line": self.line, "column": self.column}
|
|
75
|
+
if self.file_path else None
|
|
76
|
+
),
|
|
77
|
+
"confidence": self.confidence,
|
|
78
|
+
"metadata": self.metadata,
|
|
79
|
+
}
|
|
80
|
+
if self.start_byte is not None:
|
|
81
|
+
result["start_byte"] = self.start_byte
|
|
82
|
+
if self.end_byte is not None:
|
|
83
|
+
result["end_byte"] = self.end_byte
|
|
84
|
+
if self.strategy:
|
|
85
|
+
result["strategy"] = self.strategy
|
|
86
|
+
return result
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Graph node definitions for call graph analysis.
|
|
2
|
+
|
|
3
|
+
Represents functions, classes, modules as nodes in the dependency graph.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Optional, List, Dict, Any, Set
|
|
11
|
+
from enum import Enum, auto
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class NodeType(Enum):
|
|
15
|
+
"""Types of nodes in the call graph."""
|
|
16
|
+
FUNCTION = auto()
|
|
17
|
+
CLASS = auto()
|
|
18
|
+
MODULE = auto()
|
|
19
|
+
VARIABLE = auto()
|
|
20
|
+
IMPORT = auto()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class GraphNode:
|
|
25
|
+
"""A node in the call graph.
|
|
26
|
+
|
|
27
|
+
Equivalent to a node in Rust's petgraph or similar graph library.
|
|
28
|
+
"""
|
|
29
|
+
id: str # Unique identifier: "module:Class.method" (P1.10: ':' splits module from qualname; module nodes have no ':')
|
|
30
|
+
name: str
|
|
31
|
+
node_type: NodeType
|
|
32
|
+
module: str # Python module path
|
|
33
|
+
file_path: str
|
|
34
|
+
line: int
|
|
35
|
+
column: int
|
|
36
|
+
end_line: int = 0 # 0 = unknown (synthetic/module nodes; symbols set it from parse position)
|
|
37
|
+
|
|
38
|
+
# P1.10 step 3: human-readable dotted name used by search and display.
|
|
39
|
+
# The id is an opaque internal handle (it may contain <locals> markers);
|
|
40
|
+
# qualified_name is the CPython-style dotted form without them. Derived
|
|
41
|
+
# in __post_init__ when not supplied (e.g. loading pre-P1.10 rows).
|
|
42
|
+
qualified_name: str = ""
|
|
43
|
+
|
|
44
|
+
# Relationships
|
|
45
|
+
calls: Set[str] = field(default_factory=set) # IDs of nodes this calls
|
|
46
|
+
called_by: Set[str] = field(default_factory=set) # IDs of nodes that call this
|
|
47
|
+
contains: Set[str] = field(default_factory=set) # For classes/modules containing functions
|
|
48
|
+
contained_in: Optional[str] = None # Parent container
|
|
49
|
+
|
|
50
|
+
# Metadata
|
|
51
|
+
is_async: bool = False
|
|
52
|
+
is_decorated: bool = False
|
|
53
|
+
decorators: List[str] = field(default_factory=list)
|
|
54
|
+
docstring: Optional[str] = None
|
|
55
|
+
signature: Optional[str] = None
|
|
56
|
+
language: str = "python"
|
|
57
|
+
|
|
58
|
+
# Analysis flags
|
|
59
|
+
is_entry_point: bool = False # e.g., route handlers, main()
|
|
60
|
+
is_test: bool = False # test_* or *_test functions
|
|
61
|
+
metadata: Dict[str, Any] = field(default_factory=dict) # arbitrary tags (e.g. synthetic=exec)
|
|
62
|
+
|
|
63
|
+
def __post_init__(self) -> None:
|
|
64
|
+
if not self.qualified_name:
|
|
65
|
+
# ':' module/qualname split → dotted display form; <locals>
|
|
66
|
+
# markers dropped. Two distinct ids (a:b.c vs a.b:c) may share a
|
|
67
|
+
# qualified_name — that is the ambiguity search reports honestly.
|
|
68
|
+
display_id = re.sub(
|
|
69
|
+
r"\([^)]*\)", "", self.id.split("::", 1)[-1]
|
|
70
|
+
).rstrip(".")
|
|
71
|
+
self.qualified_name = display_id.replace(":", ".").replace(".<locals>", "")
|
|
72
|
+
|
|
73
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
74
|
+
return {
|
|
75
|
+
"id": self.id,
|
|
76
|
+
"qualified_name": self.qualified_name,
|
|
77
|
+
"name": self.name,
|
|
78
|
+
"type": self.node_type.name,
|
|
79
|
+
"module": self.module.split("::", 1)[-1],
|
|
80
|
+
"file": self.file_path,
|
|
81
|
+
"position": {"line": self.line, "column": self.column, "end_line": self.end_line},
|
|
82
|
+
"calls": list(self.calls),
|
|
83
|
+
"called_by": list(self.called_by),
|
|
84
|
+
"is_async": self.is_async,
|
|
85
|
+
"is_entry_point": self.is_entry_point,
|
|
86
|
+
"is_test": self.is_test,
|
|
87
|
+
"language": self.language,
|
|
88
|
+
"signature": self.signature,
|
|
89
|
+
"metadata": self.metadata,
|
|
90
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Harness layer: gets the MCP tools into the agent's hands.
|
|
2
|
+
|
|
3
|
+
`setup` — write/merge client configs (.mcp.json, .cursor/mcp.json,
|
|
4
|
+
.codex/config.toml, ~/.gemini/.../mcp_config.json) and install
|
|
5
|
+
the skill file;
|
|
6
|
+
`doctor` verifies the wiring end-to-end.
|
|
7
|
+
`watch` — hash-based polling loop that keeps the index fresh while editing.
|
|
8
|
+
|
|
9
|
+
See HARNESS.md for the design rationale.
|
|
10
|
+
"""
|