adl-toolkit 0.0.0-stage → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTALL_TOOLKIT.md +163 -0
- package/README.md +269 -2
- package/adl_drift/__init__.py +6 -0
- package/adl_drift/cli/__init__.py +1 -0
- package/adl_drift/cli/commands.py +993 -0
- package/adl_drift/cli/main.py +744 -0
- package/adl_drift/cortex/__init__.py +1 -0
- package/adl_drift/cortex/embeddings.py +96 -0
- package/adl_drift/cortex/memory.py +1749 -0
- package/adl_drift/cortex/patterns.py +89 -0
- package/adl_drift/cortex/telemetry.py +192 -0
- package/adl_drift/detectors/__init__.py +1 -0
- package/adl_drift/detectors/async_patterns.py +115 -0
- package/adl_drift/detectors/base.py +121 -0
- package/adl_drift/detectors/config.py +122 -0
- package/adl_drift/detectors/data_access.py +128 -0
- package/adl_drift/detectors/decorators.py +122 -0
- package/adl_drift/detectors/dynamic_patterns.py +260 -0
- package/adl_drift/detectors/errors.py +134 -0
- package/adl_drift/detectors/imports.py +132 -0
- package/adl_drift/detectors/logging_patterns.py +108 -0
- package/adl_drift/detectors/routes.py +257 -0
- package/adl_drift/detectors/security.py +236 -0
- package/adl_drift/detectors/taint.py +406 -0
- package/adl_drift/detectors/tests.py +95 -0
- package/adl_drift/detectors/type_conventions.py +81 -0
- package/adl_drift/graph/__init__.py +1 -0
- package/adl_drift/graph/builder.py +2670 -0
- package/adl_drift/graph/cfg.py +461 -0
- package/adl_drift/graph/cluster.py +40 -0
- package/adl_drift/graph/edges.py +86 -0
- package/adl_drift/graph/node.py +90 -0
- package/adl_drift/harness/__init__.py +10 -0
- package/adl_drift/harness/setup.py +832 -0
- package/adl_drift/harness/skills/adl-drift.md +25 -0
- package/adl_drift/harness/skills/drift-antigravity.md +19 -0
- package/adl_drift/harness/skills/drift-generic.md +19 -0
- package/adl_drift/harness/watch.py +84 -0
- package/adl_drift/lang/__init__.py +8 -0
- package/adl_drift/lang/base.py +464 -0
- package/adl_drift/lang/java/__init__.py +6 -0
- package/adl_drift/lang/java/backend.py +97 -0
- package/adl_drift/lang/java/cfg.py +1424 -0
- package/adl_drift/lang/java/config.py +707 -0
- package/adl_drift/lang/java/detectors/__init__.py +29 -0
- package/adl_drift/lang/java/detectors/additional.py +445 -0
- package/adl_drift/lang/java/detectors/base.py +154 -0
- package/adl_drift/lang/java/detectors/data_access.py +148 -0
- package/adl_drift/lang/java/detectors/imports.py +53 -0
- package/adl_drift/lang/java/detectors/routes.py +159 -0
- package/adl_drift/lang/java/detectors/security.py +386 -0
- package/adl_drift/lang/java/detectors/tests.py +74 -0
- package/adl_drift/lang/java/extractor.py +582 -0
- package/adl_drift/lang/java/ids.py +99 -0
- package/adl_drift/lang/java/parser.py +49 -0
- package/adl_drift/lang/java/resolution/__init__.py +1 -0
- package/adl_drift/lang/java/resolution/constants.py +1619 -0
- package/adl_drift/lang/java/resolution/constructors.py +278 -0
- package/adl_drift/lang/java/resolution/context.py +305 -0
- package/adl_drift/lang/java/resolution/dispatch.py +540 -0
- package/adl_drift/lang/java/resolution/functional.py +1318 -0
- package/adl_drift/lang/java/resolution/generated.py +95 -0
- package/adl_drift/lang/java/resolution/model.py +153 -0
- package/adl_drift/lang/java/resolution/orchestration.py +866 -0
- package/adl_drift/lang/java/resolution/overloads.py +1275 -0
- package/adl_drift/lang/java/resolution/scopes.py +1111 -0
- package/adl_drift/lang/java/resolution/types.py +2947 -0
- package/adl_drift/lang/java/resolver.py +1401 -0
- package/adl_drift/lang/java/scopes.py +69 -0
- package/adl_drift/lang/java/taint.py +500 -0
- package/adl_drift/lang/java/type_provenance.py +342 -0
- package/adl_drift/lang/python/__init__.py +8 -0
- package/adl_drift/lang/python/backend.py +94 -0
- package/adl_drift/lang/python/cfg.py +537 -0
- package/adl_drift/lang/python/extractor.py +454 -0
- package/adl_drift/lang/python/parser.py +41 -0
- package/adl_drift/lang/python/resolver.py +1789 -0
- package/adl_drift/lang/python/taint.py +365 -0
- package/adl_drift/lang/registry.py +117 -0
- package/adl_drift/parser/__init__.py +1 -0
- package/adl_drift/parser/cache.py +38 -0
- package/adl_drift/parser/core.py +33 -0
- package/adl_drift/parser/extractor.py +17 -0
- package/adl_drift/parser/literal_eval.py +326 -0
- package/adl_drift/scanner/__init__.py +1 -0
- package/adl_drift/scanner/discovery.py +338 -0
- package/adl_drift/scanner/incremental.py +261 -0
- package/adl_drift/scanner/parallel.py +224 -0
- package/adl_drift/server/__init__.py +1 -0
- package/adl_drift/server/handlers.py +2849 -0
- package/adl_drift/server/mcp_server.py +127 -0
- package/adl_drift/server/mcp_stdio.py +335 -0
- package/adl_drift/server/protocol.py +71 -0
- package/adl_drift/server/static/index.html +696 -0
- package/adl_drift/server/web.py +350 -0
- package/adl_drift/utils/__init__.py +1 -0
- package/adl_drift/utils/config.py +77 -0
- package/adl_drift/utils/fswalk.py +52 -0
- package/adl_drift/utils/types.py +345 -0
- package/bin/adl-drift.mjs +35 -0
- package/bin/adl-toolkit.mjs +488 -0
- package/package.json +29 -4
- package/pyproject.toml +49 -0
- package/scripts/install_toolkit.py +3584 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Security pattern detector — high-signal single-file checks.
|
|
2
|
+
|
|
3
|
+
Flags eval/exec/compile, subprocess shell=True, unsafe pickle/yaml,
|
|
4
|
+
os.system/popen, hardcoded secrets, and asserts in non-test code.
|
|
5
|
+
|
|
6
|
+
P2.18 integration: when def-use facts are available (passed in via
|
|
7
|
+
``def_use_facts`` on the detector instance), dangerous-builtin findings
|
|
8
|
+
are upgraded to CRITICAL (taint-confirmed) or downgraded to LOW (literal-
|
|
9
|
+
only arguments). Facts are populated by the scan pipeline after the
|
|
10
|
+
CFG+reaching-defs pass.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Dict, List, Optional
|
|
18
|
+
|
|
19
|
+
from tree_sitter import Node, Tree
|
|
20
|
+
|
|
21
|
+
from adl_drift.detectors.base import BaseDetector
|
|
22
|
+
from adl_drift.utils.types import Pattern, PatternCategory
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
SECRET_RE = re.compile(
|
|
26
|
+
r"(?i)(api[_-]?key|secret|token|password|passwd|pwd)\s*=\s*['\"]([^'\"]{8,})['\"]"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class SecurityDetector(BaseDetector):
|
|
31
|
+
"""Detects high-confidence security anti-patterns in a single file."""
|
|
32
|
+
|
|
33
|
+
DANGEROUS_BUILTINS = {"eval", "exec", "compile"}
|
|
34
|
+
OS_SHELL = {"os.system", "os.popen"}
|
|
35
|
+
SUBPROCESS = {"subprocess.call", "subprocess.run", "subprocess.Popen",
|
|
36
|
+
"subprocess.check_call", "subprocess.check_output"}
|
|
37
|
+
PICKLE_LOADS = {"pickle.load", "pickle.loads", "cPickle.load", "cPickle.loads"}
|
|
38
|
+
|
|
39
|
+
def __init__(self, parser: Any) -> None:
|
|
40
|
+
super().__init__(parser)
|
|
41
|
+
# P2.18: populated by scan pipeline after CFG pass.
|
|
42
|
+
# Maps function_id → {binding_name → List[DefUseFact]}.
|
|
43
|
+
self.def_use_facts: Dict[str, Dict[str, List[Any]]] = {}
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def category(self) -> PatternCategory:
|
|
47
|
+
return PatternCategory.SECURITY
|
|
48
|
+
|
|
49
|
+
def detect(self, tree: Tree, source_bytes: bytes, file_path: Path) -> List[Pattern]:
|
|
50
|
+
patterns: List[Pattern] = []
|
|
51
|
+
is_test_file = self._is_test_file(file_path)
|
|
52
|
+
|
|
53
|
+
for node in self._walk(tree.root_node):
|
|
54
|
+
if node.type == "call":
|
|
55
|
+
patterns.extend(self._check_call(node, source_bytes, file_path))
|
|
56
|
+
elif node.type == "assert_statement" and not is_test_file:
|
|
57
|
+
patterns.append(self._make_pattern(
|
|
58
|
+
name="assert_in_prod",
|
|
59
|
+
description="assert in non-test code (stripped under -O)",
|
|
60
|
+
node=node, file_path=file_path, confidence=0.3,
|
|
61
|
+
metadata={"severity": "LOW"},
|
|
62
|
+
))
|
|
63
|
+
|
|
64
|
+
# Hardcoded secrets — regex over source text
|
|
65
|
+
try:
|
|
66
|
+
text = source_bytes.decode("utf-8", errors="replace")
|
|
67
|
+
except Exception:
|
|
68
|
+
text = ""
|
|
69
|
+
for m in SECRET_RE.finditer(text):
|
|
70
|
+
line = text.count("\n", 0, m.start()) + 1
|
|
71
|
+
patterns.append(Pattern(
|
|
72
|
+
category=self.category,
|
|
73
|
+
name="hardcoded_secret",
|
|
74
|
+
description=f"Possible hardcoded secret: {m.group(1)}",
|
|
75
|
+
position=self._line_position(line),
|
|
76
|
+
file_path=file_path,
|
|
77
|
+
confidence=0.6,
|
|
78
|
+
metadata={"severity": "MEDIUM", "key_name": m.group(1)},
|
|
79
|
+
))
|
|
80
|
+
|
|
81
|
+
return patterns
|
|
82
|
+
|
|
83
|
+
def _check_call(self, node: Node, source: bytes, file_path: Path) -> List[Pattern]:
|
|
84
|
+
out: List[Pattern] = []
|
|
85
|
+
func_node = node.child_by_field_name("function")
|
|
86
|
+
if func_node is None:
|
|
87
|
+
return out
|
|
88
|
+
func_text = self.parser.node_text(func_node, source).strip()
|
|
89
|
+
|
|
90
|
+
if func_text in self.DANGEROUS_BUILTINS:
|
|
91
|
+
severity, confidence, extra = self._refine_dangerous_builtin(
|
|
92
|
+
func_text, node, source
|
|
93
|
+
)
|
|
94
|
+
out.append(self._make_pattern(
|
|
95
|
+
name=f"dangerous_{func_text}",
|
|
96
|
+
description=f"{func_text}() executes arbitrary code",
|
|
97
|
+
node=node, file_path=file_path, confidence=confidence,
|
|
98
|
+
metadata={"severity": severity, "function": func_text, **extra},
|
|
99
|
+
))
|
|
100
|
+
return out
|
|
101
|
+
|
|
102
|
+
if func_text in self.OS_SHELL:
|
|
103
|
+
out.append(self._make_pattern(
|
|
104
|
+
name="os_shell",
|
|
105
|
+
description=f"{func_text}() runs a shell command",
|
|
106
|
+
node=node, file_path=file_path, confidence=0.9,
|
|
107
|
+
metadata={"severity": "HIGH", "function": func_text},
|
|
108
|
+
))
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
if func_text in self.SUBPROCESS:
|
|
112
|
+
if self._has_shell_true(node, source):
|
|
113
|
+
out.append(self._make_pattern(
|
|
114
|
+
name="subprocess_shell_true",
|
|
115
|
+
description=f"{func_text}(shell=True) shell-injection risk",
|
|
116
|
+
node=node, file_path=file_path, confidence=0.9,
|
|
117
|
+
metadata={"severity": "HIGH", "function": func_text},
|
|
118
|
+
))
|
|
119
|
+
return out
|
|
120
|
+
|
|
121
|
+
if func_text in self.PICKLE_LOADS:
|
|
122
|
+
out.append(self._make_pattern(
|
|
123
|
+
name="unsafe_pickle",
|
|
124
|
+
description=f"{func_text}() deserializes arbitrary code",
|
|
125
|
+
node=node, file_path=file_path, confidence=0.85,
|
|
126
|
+
metadata={"severity": "HIGH", "function": func_text},
|
|
127
|
+
))
|
|
128
|
+
return out
|
|
129
|
+
|
|
130
|
+
if func_text in {"yaml.load"} and not self._has_safe_loader(node, source):
|
|
131
|
+
out.append(self._make_pattern(
|
|
132
|
+
name="unsafe_yaml_load",
|
|
133
|
+
description="yaml.load() without SafeLoader",
|
|
134
|
+
node=node, file_path=file_path, confidence=0.85,
|
|
135
|
+
metadata={"severity": "HIGH", "function": func_text},
|
|
136
|
+
))
|
|
137
|
+
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
def _refine_dangerous_builtin(
|
|
141
|
+
self, func_text: str, call_node: Node, source: bytes
|
|
142
|
+
):
|
|
143
|
+
"""P2.18: use def-use facts to refine severity for eval/exec/compile.
|
|
144
|
+
|
|
145
|
+
Returns (severity, confidence, extra_metadata).
|
|
146
|
+
|
|
147
|
+
Rules:
|
|
148
|
+
- Argument is a literal node directly → LOW (no data-flow needed)
|
|
149
|
+
- Argument is an identifier AND all reaching defs are literals
|
|
150
|
+
→ LOW ("always_literal" in metadata)
|
|
151
|
+
- Argument is an identifier AND any reaching def is tainted
|
|
152
|
+
→ CRITICAL ("taint_source" in metadata, highest priority)
|
|
153
|
+
- Otherwise → HIGH (default, no facts or mixed/unknown defs)
|
|
154
|
+
"""
|
|
155
|
+
from adl_drift.graph.cfg import _is_literal as cfg_is_literal
|
|
156
|
+
|
|
157
|
+
args = call_node.child_by_field_name("arguments")
|
|
158
|
+
first_arg = None
|
|
159
|
+
if args:
|
|
160
|
+
for child in args.children:
|
|
161
|
+
if child.type not in ("(", ")", ",", "comment"):
|
|
162
|
+
first_arg = child
|
|
163
|
+
break
|
|
164
|
+
|
|
165
|
+
# Direct literal argument — no data-flow needed
|
|
166
|
+
if first_arg is not None and cfg_is_literal(first_arg):
|
|
167
|
+
return "LOW", 0.5, {"data_flow": "literal_arg"}
|
|
168
|
+
|
|
169
|
+
# Identifier argument — look up def-use facts
|
|
170
|
+
if first_arg is not None and first_arg.type == "identifier" and self.def_use_facts:
|
|
171
|
+
arg_name = (
|
|
172
|
+
first_arg.text.decode("utf-8", errors="replace")
|
|
173
|
+
if first_arg.text else ""
|
|
174
|
+
)
|
|
175
|
+
if arg_name:
|
|
176
|
+
enclosing_fid = self._find_enclosing_function_id(call_node)
|
|
177
|
+
facts_by_func = self.def_use_facts.get(enclosing_fid, {})
|
|
178
|
+
arg_facts = facts_by_func.get(arg_name, [])
|
|
179
|
+
if arg_facts:
|
|
180
|
+
tainted = [f for f in arg_facts if f.taint_source]
|
|
181
|
+
all_literal = all(f.is_literal for f in arg_facts)
|
|
182
|
+
if tainted:
|
|
183
|
+
return "CRITICAL", 0.95, {
|
|
184
|
+
"data_flow": "taint_confirmed",
|
|
185
|
+
"taint_source": tainted[0].taint_source,
|
|
186
|
+
}
|
|
187
|
+
if all_literal:
|
|
188
|
+
return "LOW", 0.5, {"data_flow": "always_literal"}
|
|
189
|
+
|
|
190
|
+
return "HIGH", 0.9, {}
|
|
191
|
+
|
|
192
|
+
def _find_enclosing_function_id(self, node: Node) -> str:
|
|
193
|
+
"""Walk up the CST to find the enclosing function's name (best effort)."""
|
|
194
|
+
cur = node.parent
|
|
195
|
+
parts: List[str] = []
|
|
196
|
+
while cur is not None:
|
|
197
|
+
if cur.type in ("function_definition", "async_function_definition"):
|
|
198
|
+
name_node = cur.child_by_field_name("name")
|
|
199
|
+
if name_node and name_node.text:
|
|
200
|
+
parts.append(name_node.text.decode("utf-8", errors="replace"))
|
|
201
|
+
cur = cur.parent
|
|
202
|
+
parts.reverse()
|
|
203
|
+
return ".".join(parts) if parts else "<module>"
|
|
204
|
+
|
|
205
|
+
def _has_shell_true(self, call_node: Node, source: bytes) -> bool:
|
|
206
|
+
args = call_node.child_by_field_name("arguments")
|
|
207
|
+
if args is None:
|
|
208
|
+
return False
|
|
209
|
+
for child in args.children:
|
|
210
|
+
if child.type == "keyword_argument":
|
|
211
|
+
name_node = child.child_by_field_name("name")
|
|
212
|
+
val_node = child.child_by_field_name("value")
|
|
213
|
+
if name_node is not None and val_node is not None:
|
|
214
|
+
if self.parser.node_text(name_node, source) == "shell" and \
|
|
215
|
+
self.parser.node_text(val_node, source) == "True":
|
|
216
|
+
return True
|
|
217
|
+
return False
|
|
218
|
+
|
|
219
|
+
def _has_safe_loader(self, call_node: Node, source: bytes) -> bool:
|
|
220
|
+
text = self.parser.node_text(call_node, source)
|
|
221
|
+
return "SafeLoader" in text or "safe_load" in text
|
|
222
|
+
|
|
223
|
+
@staticmethod
|
|
224
|
+
def _is_test_file(file_path: Path) -> bool:
|
|
225
|
+
n = file_path.name
|
|
226
|
+
return n.startswith("test_") or n.endswith("_test.py") or "tests" in file_path.parts
|
|
227
|
+
|
|
228
|
+
@staticmethod
|
|
229
|
+
def _line_position(line: int):
|
|
230
|
+
from adl_drift.utils.types import SourcePosition
|
|
231
|
+
return SourcePosition(line=line, column=0, end_line=line, end_column=0)
|
|
232
|
+
|
|
233
|
+
def _walk(self, node: Node):
|
|
234
|
+
yield node
|
|
235
|
+
for child in node.children:
|
|
236
|
+
yield from self._walk(child)
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
"""Taint analysis — source→sink tracking (P2.19, generalized to a
|
|
2
|
+
per-language seam in P4.6 — see PROBLEMS.md P4.6).
|
|
3
|
+
|
|
4
|
+
Two layers:
|
|
5
|
+
|
|
6
|
+
- **Intra-function** (`TaintStrategy.analyze_one`, per language — Python's
|
|
7
|
+
is `lang/python/taint.py::PythonTaintStrategy`, Java's is `lang/java/
|
|
8
|
+
taint.py::JavaTaintStrategy`): flow-insensitive taint propagation over
|
|
9
|
+
one function body, driven by the source/sink tables on a `CFGConfig`
|
|
10
|
+
(`lang/base.py`). A binding is tainted if it is assigned (directly or
|
|
11
|
+
transitively) from a known taint source or from another tainted
|
|
12
|
+
binding. When a tainted binding reaches a sink call-site argument → a
|
|
13
|
+
`TaintPath` is emitted. This is the CST-coupled "mechanics" half —
|
|
14
|
+
mirrors how P4.4 split `CallResolver` out of the shared resolution
|
|
15
|
+
driver.
|
|
16
|
+
|
|
17
|
+
- **Cross-function** (`TaintAnalyzer.analyze`): a worklist fixpoint over
|
|
18
|
+
the call graph, fully language-agnostic — it never inspects a CST node,
|
|
19
|
+
only `TaintStrategy.analyze_one`'s `(tainted, TaintPath, _OutCall)`
|
|
20
|
+
results. When a function passes a tainted argument to a callee, the
|
|
21
|
+
callee's matching parameter becomes a taint entry; the callee is
|
|
22
|
+
re-analyzed with that seed. Callees are resolved by simple name against
|
|
23
|
+
the function index (so it follows calls across modules), over-approximating
|
|
24
|
+
on ambiguity — taint recall is best-effort by design.
|
|
25
|
+
|
|
26
|
+
Identity (P1.10 boundary rule): taint state is keyed by binding *name* scoped to
|
|
27
|
+
the enclosing function id, never by a global dotted name — the data-flow layer
|
|
28
|
+
stays immune to the node-id collision classes tracked in P1.10.
|
|
29
|
+
|
|
30
|
+
Limitation: cross-function taint only follows calls between functions whose CSTs
|
|
31
|
+
are present in the current scan (all of them on a full scan; the changed set on
|
|
32
|
+
an incremental scan). This mirrors the per-file best-effort design of the
|
|
33
|
+
def-use pass.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
from collections import defaultdict, deque
|
|
39
|
+
from dataclasses import dataclass, field
|
|
40
|
+
from pathlib import Path
|
|
41
|
+
from typing import Any, Dict, List, Optional, Set, Tuple
|
|
42
|
+
|
|
43
|
+
from tree_sitter import Node
|
|
44
|
+
|
|
45
|
+
from adl_drift.graph.edges import EdgeType
|
|
46
|
+
from adl_drift.lang.base import CFGConfig
|
|
47
|
+
from adl_drift.utils.types import SourcePosition
|
|
48
|
+
|
|
49
|
+
MAX_FIXPOINT_ITERATIONS = 12
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# ─── public data type ───────────────────────────────────────────────────────
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class TaintPath:
|
|
56
|
+
"""A confirmed source→sink data-flow path inside one function.
|
|
57
|
+
|
|
58
|
+
``binding_name`` is the tainted variable handed to the sink (``None`` when a
|
|
59
|
+
source expression is passed inline, e.g. ``eval(request.GET["x"])``).
|
|
60
|
+
``cross_function`` is True when the taint entered through a parameter rather
|
|
61
|
+
than a source assigned in this function body.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
function_id: str
|
|
65
|
+
file_path: str
|
|
66
|
+
sink: str
|
|
67
|
+
sink_category: str
|
|
68
|
+
source_label: str
|
|
69
|
+
binding_name: Optional[str]
|
|
70
|
+
sink_site: SourcePosition
|
|
71
|
+
chain: List[str] = field(default_factory=list)
|
|
72
|
+
confidence: float = 0.9
|
|
73
|
+
cross_function: bool = False
|
|
74
|
+
|
|
75
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
76
|
+
return {
|
|
77
|
+
"function_id": self.function_id,
|
|
78
|
+
"file_path": self.file_path,
|
|
79
|
+
"sink": self.sink,
|
|
80
|
+
"sink_category": self.sink_category,
|
|
81
|
+
"source_label": self.source_label,
|
|
82
|
+
"binding_name": self.binding_name,
|
|
83
|
+
"sink_site": self.sink_site.to_dict(),
|
|
84
|
+
"chain": list(self.chain),
|
|
85
|
+
"confidence": self.confidence,
|
|
86
|
+
"cross_function": self.cross_function,
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def from_dict(cls, d: Dict[str, Any]) -> "TaintPath":
|
|
91
|
+
return cls(
|
|
92
|
+
function_id=d["function_id"],
|
|
93
|
+
file_path=d["file_path"],
|
|
94
|
+
sink=d["sink"],
|
|
95
|
+
sink_category=d["sink_category"],
|
|
96
|
+
source_label=d["source_label"],
|
|
97
|
+
binding_name=d.get("binding_name"),
|
|
98
|
+
sink_site=SourcePosition.from_dict(d["sink_site"]),
|
|
99
|
+
chain=list(d.get("chain", [])),
|
|
100
|
+
confidence=d.get("confidence", 0.9),
|
|
101
|
+
cross_function=d.get("cross_function", False),
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass
|
|
106
|
+
class _OutCall:
|
|
107
|
+
"""An outgoing call site from an analyzed function whose arguments carry
|
|
108
|
+
taint — positional index / keyword name of every tainted argument.
|
|
109
|
+
Shared shape every `TaintStrategy.analyze_one` implementation returns,
|
|
110
|
+
consumed by the language-agnostic cross-function worklist below.
|
|
111
|
+
|
|
112
|
+
`site_line` (1-indexed, matching `GraphEdge.line`'s convention) is the
|
|
113
|
+
call node's own source line — call-site identity, so `_resolve_callees`
|
|
114
|
+
can tell *this specific call* apart from a different call to a
|
|
115
|
+
same-named method elsewhere in the same function (F-03, PROBLEMS.md
|
|
116
|
+
P4.6: `good.process(tainted); bad.process("static");` in one function
|
|
117
|
+
must not both get resolved from a single `_OutCall`). `None` for a
|
|
118
|
+
language whose strategy doesn't supply it — `_resolve_callees` then
|
|
119
|
+
falls back to matching by name alone, same as before this field existed.
|
|
120
|
+
|
|
121
|
+
`site_column` (0-indexed, matching `GraphEdge.column`'s / tree-sitter
|
|
122
|
+
`start_point[1]`'s convention) is the call node's own source column —
|
|
123
|
+
needed on top of `site_line` because two distinct calls to the same
|
|
124
|
+
simple name can land on the *same* line (`good.process(q);
|
|
125
|
+
bad.process("static");` written on one line, or simply two call
|
|
126
|
+
expressions used as arguments to each other). Matching by line alone
|
|
127
|
+
cannot tell those apart and would over-approximate back to the
|
|
128
|
+
F-03 bug it was meant to fix. `None` for a language/site whose strategy
|
|
129
|
+
doesn't supply it — `_resolve_callees` then falls back to line-only,
|
|
130
|
+
then name-only, matching."""
|
|
131
|
+
|
|
132
|
+
callee: str
|
|
133
|
+
pos: Set[int]
|
|
134
|
+
kw: Set[str]
|
|
135
|
+
site_line: Optional[int] = None
|
|
136
|
+
site_column: Optional[int] = None
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# ─── per-language intra-function mechanics (P4.6) ────────────────────────────
|
|
140
|
+
|
|
141
|
+
class TaintStrategy:
|
|
142
|
+
"""Per-language intra-function taint mechanics (P4.6, PROBLEMS.md).
|
|
143
|
+
|
|
144
|
+
Owns everything CST-coupled about taint tracking for one language: how a
|
|
145
|
+
binding gets its parameter list, and how one function body propagates
|
|
146
|
+
taint from sources to sinks. `TaintAnalyzer` (below) drives this
|
|
147
|
+
strategy inside a fully language-agnostic cross-function worklist —
|
|
148
|
+
mirrors how `graph/builder.py::CallGraphBuilder` drives a `CallResolver`
|
|
149
|
+
(P4.4).
|
|
150
|
+
|
|
151
|
+
`lang/python/taint.py::PythonTaintStrategy` and `lang/java/taint.py::
|
|
152
|
+
JavaTaintStrategy` are the two implementations. Never constructed
|
|
153
|
+
directly — obtained via `CallGraphBuilder._get_taint_strategy()`, which
|
|
154
|
+
builds one lazily via `CFGConfig.taint_strategy_factory`.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
def __init__(self, config: CFGConfig) -> None:
|
|
158
|
+
self.config = config
|
|
159
|
+
|
|
160
|
+
def param_names(self, func_node: Node, source: bytes) -> List[str]:
|
|
161
|
+
"""Ordered parameter identifiers for `func_node` (includes an
|
|
162
|
+
explicit receiver name where the language has one, e.g. Python's
|
|
163
|
+
`self`/`cls` — Java has none, `this` is never a formal parameter)."""
|
|
164
|
+
raise NotImplementedError
|
|
165
|
+
|
|
166
|
+
def analyze_one(
|
|
167
|
+
self,
|
|
168
|
+
func_node: Node,
|
|
169
|
+
source: bytes,
|
|
170
|
+
function_id: str,
|
|
171
|
+
file_path: str,
|
|
172
|
+
seed_params: Set[str],
|
|
173
|
+
) -> Tuple[Set[str], List[TaintPath], List[_OutCall]]:
|
|
174
|
+
"""Flow-insensitive taint over one function body.
|
|
175
|
+
|
|
176
|
+
Returns ``(tainted_bindings, sink_paths, out_calls)``."""
|
|
177
|
+
raise NotImplementedError
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# ─── graph-level driver (intra + cross function; language-agnostic) ─────────
|
|
181
|
+
|
|
182
|
+
@dataclass
|
|
183
|
+
class _FuncMeta:
|
|
184
|
+
node_id: str
|
|
185
|
+
name: str
|
|
186
|
+
func_node: Node
|
|
187
|
+
source: bytes
|
|
188
|
+
file_path: str
|
|
189
|
+
params: List[str]
|
|
190
|
+
is_method: bool
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class TaintAnalyzer:
|
|
194
|
+
"""Runs intra- and cross-function taint over a built graph + parse trees."""
|
|
195
|
+
|
|
196
|
+
def analyze(self, builder: Any, parses: Dict[Path, Any]) -> List[TaintPath]:
|
|
197
|
+
strategy = builder._get_taint_strategy()
|
|
198
|
+
metas = self._build_func_index(builder, parses, strategy)
|
|
199
|
+
if not metas:
|
|
200
|
+
return []
|
|
201
|
+
|
|
202
|
+
by_name: Dict[str, List[str]] = defaultdict(list)
|
|
203
|
+
for nid, m in metas.items():
|
|
204
|
+
by_name[m.name].append(nid)
|
|
205
|
+
|
|
206
|
+
seeds: Dict[str, Set[str]] = {nid: set() for nid in metas}
|
|
207
|
+
worklist: deque = deque(metas.keys())
|
|
208
|
+
in_wl: Set[str] = set(metas)
|
|
209
|
+
iterations = 0
|
|
210
|
+
max_iter = len(metas) * MAX_FIXPOINT_ITERATIONS + len(metas)
|
|
211
|
+
|
|
212
|
+
# Cache the last out_calls per node to propagate without re-emitting paths.
|
|
213
|
+
while worklist and iterations < max_iter:
|
|
214
|
+
iterations += 1
|
|
215
|
+
nid = worklist.popleft()
|
|
216
|
+
in_wl.discard(nid)
|
|
217
|
+
m = metas[nid]
|
|
218
|
+
_, _, out_calls = strategy.analyze_one(
|
|
219
|
+
m.func_node, m.source, nid, m.file_path, seeds[nid]
|
|
220
|
+
)
|
|
221
|
+
for oc in out_calls:
|
|
222
|
+
for cand_id in self._resolve_callees(builder, nid, oc, by_name, metas):
|
|
223
|
+
cand = metas[cand_id]
|
|
224
|
+
added = self._map_taint_to_params(cand, oc, seeds[cand_id])
|
|
225
|
+
if added:
|
|
226
|
+
seeds[cand_id] |= added
|
|
227
|
+
if cand_id not in in_wl:
|
|
228
|
+
worklist.append(cand_id)
|
|
229
|
+
in_wl.add(cand_id)
|
|
230
|
+
|
|
231
|
+
# Final pass: emit sink paths with the converged seeds.
|
|
232
|
+
paths: List[TaintPath] = []
|
|
233
|
+
for nid, m in metas.items():
|
|
234
|
+
_, sink_paths, _ = strategy.analyze_one(
|
|
235
|
+
m.func_node, m.source, nid, m.file_path, seeds[nid]
|
|
236
|
+
)
|
|
237
|
+
paths.extend(sink_paths)
|
|
238
|
+
return paths
|
|
239
|
+
|
|
240
|
+
@staticmethod
|
|
241
|
+
def _resolve_callees(
|
|
242
|
+
builder: Any,
|
|
243
|
+
caller_id: str,
|
|
244
|
+
oc: "_OutCall",
|
|
245
|
+
by_name: Dict[str, List[str]],
|
|
246
|
+
metas: Dict[str, "_FuncMeta"],
|
|
247
|
+
) -> List[str]:
|
|
248
|
+
"""Candidate target function ids for the call site *oc* made inside
|
|
249
|
+
*caller_id*.
|
|
250
|
+
|
|
251
|
+
Prefers the call graph's own resolved `CALLS` edges out of this
|
|
252
|
+
specific caller (P4.4 overload/receiver-type resolution, P4.12
|
|
253
|
+
interface/DI fan-out) over blind simple-name fan-out — otherwise two
|
|
254
|
+
unrelated classes with a same-named method (`Good.process`/`Bad.
|
|
255
|
+
process`) would both get seeded just because *some* function in the
|
|
256
|
+
codebase resolves a call to one of them.
|
|
257
|
+
|
|
258
|
+
When `caller_id` has more than one resolved `CALLS` edge for this
|
|
259
|
+
callee name (e.g. two *different* call sites in the same function,
|
|
260
|
+
`good.process(tainted); bad.process("static");` — both legitimately
|
|
261
|
+
resolved, to different targets), matching on name alone would still
|
|
262
|
+
conflate them. `oc.site_line` (the call node's own source line)
|
|
263
|
+
narrows this down to the specific call site's own edge(s) — still
|
|
264
|
+
possibly more than one, for genuine same-site fan-out (P4.12
|
|
265
|
+
interface/DI dispatch), which is correct to keep.
|
|
266
|
+
|
|
267
|
+
Two distinct same-named calls can also share one line (`good.
|
|
268
|
+
process(q); bad.process("static");` on one physical line — a real
|
|
269
|
+
edge case reported against the line-only version of this fix,
|
|
270
|
+
PROBLEMS.md P4.6 F-03 remaining gap). So within the line match,
|
|
271
|
+
`oc.site_column` narrows further to the exact call-site node when
|
|
272
|
+
more than one candidate remains on that line; ties still fall
|
|
273
|
+
through to "by line" (all edges on that line) when the column
|
|
274
|
+
doesn't single one out. Falls back progressively — by line only,
|
|
275
|
+
then by name only — covers `site_line`/`site_column` being `None`
|
|
276
|
+
(a `TaintStrategy` or resolver edge that doesn't supply them) and
|
|
277
|
+
any resolver/extraction position mismatch — recall over precision,
|
|
278
|
+
same as the no-CALLS-edge-at-all fallback below.
|
|
279
|
+
|
|
280
|
+
Falls back to `by_name` only when the resolver produced no `CALLS`
|
|
281
|
+
edge at all from this caller for this callee name (e.g. an
|
|
282
|
+
unresolved dynamic dispatch) — recall over precision in that case,
|
|
283
|
+
matching this worklist's existing best-effort design for genuinely
|
|
284
|
+
unresolved calls.
|
|
285
|
+
"""
|
|
286
|
+
nodes = getattr(builder, "_nodes", None)
|
|
287
|
+
occurrences = getattr(builder, "_edge_occurrences", None)
|
|
288
|
+
if occurrences and nodes is not None:
|
|
289
|
+
by_name_occurrences = [
|
|
290
|
+
edge for edge in occurrences
|
|
291
|
+
if edge.source == caller_id
|
|
292
|
+
and edge.edge_type == EdgeType.CALLS
|
|
293
|
+
and edge.target in metas
|
|
294
|
+
and nodes.get(edge.target) is not None
|
|
295
|
+
and nodes[edge.target].name == oc.callee
|
|
296
|
+
]
|
|
297
|
+
if by_name_occurrences:
|
|
298
|
+
if oc.site_line is not None:
|
|
299
|
+
by_line = [
|
|
300
|
+
edge for edge in by_name_occurrences
|
|
301
|
+
if edge.line == oc.site_line
|
|
302
|
+
]
|
|
303
|
+
if by_line:
|
|
304
|
+
if oc.site_column is not None:
|
|
305
|
+
by_column = [
|
|
306
|
+
edge.target for edge in by_line
|
|
307
|
+
if edge.column == oc.site_column
|
|
308
|
+
]
|
|
309
|
+
if by_column:
|
|
310
|
+
return list(dict.fromkeys(by_column))
|
|
311
|
+
return list(dict.fromkeys(edge.target for edge in by_line))
|
|
312
|
+
return list(dict.fromkeys(edge.target for edge in by_name_occurrences))
|
|
313
|
+
|
|
314
|
+
graph = getattr(builder, "graph", None)
|
|
315
|
+
if graph is not None and nodes is not None and graph.has_node(caller_id):
|
|
316
|
+
by_name_edges = [
|
|
317
|
+
(target, data)
|
|
318
|
+
for _, target, data in graph.out_edges(caller_id, data=True)
|
|
319
|
+
if data.get("type") == EdgeType.CALLS.name
|
|
320
|
+
and target in metas
|
|
321
|
+
and nodes.get(target) is not None
|
|
322
|
+
and nodes[target].name == oc.callee
|
|
323
|
+
]
|
|
324
|
+
if by_name_edges:
|
|
325
|
+
if oc.site_line is not None:
|
|
326
|
+
# `GraphEdge.to_dict()` (what `_add_edge` feeds into the
|
|
327
|
+
# NetworkX edge data dict) nests the call-site line/column
|
|
328
|
+
# under `"position"`, not as top-level keys.
|
|
329
|
+
by_site = [
|
|
330
|
+
t for t, d in by_name_edges
|
|
331
|
+
if (d.get("position") or {}).get("line") == oc.site_line
|
|
332
|
+
]
|
|
333
|
+
if by_site:
|
|
334
|
+
if oc.site_column is not None:
|
|
335
|
+
by_site_edges = [
|
|
336
|
+
(t, d) for t, d in by_name_edges
|
|
337
|
+
if (d.get("position") or {}).get("line") == oc.site_line
|
|
338
|
+
]
|
|
339
|
+
by_column = [
|
|
340
|
+
t for t, d in by_site_edges
|
|
341
|
+
if (d.get("position") or {}).get("column") == oc.site_column
|
|
342
|
+
]
|
|
343
|
+
if by_column:
|
|
344
|
+
return by_column
|
|
345
|
+
return by_site
|
|
346
|
+
return [t for t, _ in by_name_edges]
|
|
347
|
+
return by_name.get(oc.callee, [])
|
|
348
|
+
|
|
349
|
+
@staticmethod
|
|
350
|
+
def _map_taint_to_params(
|
|
351
|
+
cand: _FuncMeta, oc: _OutCall, existing: Set[str]
|
|
352
|
+
) -> Set[str]:
|
|
353
|
+
"""Map tainted call arguments to the callee's parameter names."""
|
|
354
|
+
added: Set[str] = set()
|
|
355
|
+
offset = 1 if cand.is_method else 0 # skip self/cls for method calls
|
|
356
|
+
for pos in oc.pos:
|
|
357
|
+
tgt = pos + offset
|
|
358
|
+
if 0 <= tgt < len(cand.params):
|
|
359
|
+
p = cand.params[tgt]
|
|
360
|
+
if p not in existing:
|
|
361
|
+
added.add(p)
|
|
362
|
+
for kw in oc.kw:
|
|
363
|
+
if kw in cand.params and kw not in existing:
|
|
364
|
+
added.add(kw)
|
|
365
|
+
return added
|
|
366
|
+
|
|
367
|
+
def _build_func_index(
|
|
368
|
+
self, builder: Any, parses: Dict[Path, Any], strategy: TaintStrategy,
|
|
369
|
+
) -> Dict[str, _FuncMeta]:
|
|
370
|
+
# `builder._path_to_module()` is path-derived and does not match the
|
|
371
|
+
# package-declaration-derived module a Java file's real graph nodes
|
|
372
|
+
# carry (`GraphNode.module`, set from `AnalysisResult.module_name` —
|
|
373
|
+
# see `lang/java/extractor.py`), so function ids computed from it
|
|
374
|
+
# would not line up with the actual graph node ids `builder._iter_
|
|
375
|
+
# func_cst` derives (via `CallGraphBuilder._symbol_to_id`). Reading
|
|
376
|
+
# the module straight off any already-built node for that file keeps
|
|
377
|
+
# this correct for every language; Python is unaffected since its
|
|
378
|
+
# nodes' `.module` already equals the path-derived value.
|
|
379
|
+
module_by_file: Dict[str, str] = {}
|
|
380
|
+
for node in builder._nodes.values():
|
|
381
|
+
module_by_file.setdefault(node.file_path, node.module)
|
|
382
|
+
|
|
383
|
+
receiver_self_names = builder._get_resolver().config.receiver_self_names
|
|
384
|
+
|
|
385
|
+
metas: Dict[str, _FuncMeta] = {}
|
|
386
|
+
for file_path, parse_result in parses.items():
|
|
387
|
+
try:
|
|
388
|
+
source = parse_result.source_bytes
|
|
389
|
+
fp_str = str(file_path)
|
|
390
|
+
module = module_by_file.get(fp_str) or builder._path_to_module(file_path)
|
|
391
|
+
except Exception:
|
|
392
|
+
continue
|
|
393
|
+
for func_cst, nid in builder._iter_func_cst(
|
|
394
|
+
parse_result.tree.root_node, source, module
|
|
395
|
+
):
|
|
396
|
+
params = strategy.param_names(func_cst, source)
|
|
397
|
+
is_method = bool(params) and params[0] in receiver_self_names
|
|
398
|
+
name_node = func_cst.child_by_field_name("name")
|
|
399
|
+
fname = source[name_node.start_byte:name_node.end_byte].decode(
|
|
400
|
+
"utf-8", errors="replace"
|
|
401
|
+
) if name_node else ""
|
|
402
|
+
metas[nid] = _FuncMeta(
|
|
403
|
+
node_id=nid, name=fname, func_node=func_cst, source=source,
|
|
404
|
+
file_path=fp_str, params=params, is_method=is_method,
|
|
405
|
+
)
|
|
406
|
+
return metas
|