adl-toolkit 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTALL_TOOLKIT.md +163 -0
- package/README.md +269 -2
- package/adl_drift/__init__.py +6 -0
- package/adl_drift/cli/__init__.py +1 -0
- package/adl_drift/cli/commands.py +993 -0
- package/adl_drift/cli/main.py +744 -0
- package/adl_drift/cortex/__init__.py +1 -0
- package/adl_drift/cortex/embeddings.py +96 -0
- package/adl_drift/cortex/memory.py +1749 -0
- package/adl_drift/cortex/patterns.py +89 -0
- package/adl_drift/cortex/telemetry.py +192 -0
- package/adl_drift/detectors/__init__.py +1 -0
- package/adl_drift/detectors/async_patterns.py +115 -0
- package/adl_drift/detectors/base.py +121 -0
- package/adl_drift/detectors/config.py +122 -0
- package/adl_drift/detectors/data_access.py +128 -0
- package/adl_drift/detectors/decorators.py +122 -0
- package/adl_drift/detectors/dynamic_patterns.py +260 -0
- package/adl_drift/detectors/errors.py +134 -0
- package/adl_drift/detectors/imports.py +132 -0
- package/adl_drift/detectors/logging_patterns.py +108 -0
- package/adl_drift/detectors/routes.py +257 -0
- package/adl_drift/detectors/security.py +236 -0
- package/adl_drift/detectors/taint.py +406 -0
- package/adl_drift/detectors/tests.py +95 -0
- package/adl_drift/detectors/type_conventions.py +81 -0
- package/adl_drift/graph/__init__.py +1 -0
- package/adl_drift/graph/builder.py +2670 -0
- package/adl_drift/graph/cfg.py +461 -0
- package/adl_drift/graph/cluster.py +40 -0
- package/adl_drift/graph/edges.py +86 -0
- package/adl_drift/graph/node.py +90 -0
- package/adl_drift/harness/__init__.py +10 -0
- package/adl_drift/harness/setup.py +832 -0
- package/adl_drift/harness/skills/drift-antigravity.md +19 -0
- package/adl_drift/harness/skills/drift-generic.md +19 -0
- package/adl_drift/harness/skills/drift.md +25 -0
- package/adl_drift/harness/watch.py +84 -0
- package/adl_drift/lang/__init__.py +8 -0
- package/adl_drift/lang/base.py +464 -0
- package/adl_drift/lang/java/__init__.py +6 -0
- package/adl_drift/lang/java/backend.py +97 -0
- package/adl_drift/lang/java/cfg.py +1424 -0
- package/adl_drift/lang/java/config.py +707 -0
- package/adl_drift/lang/java/detectors/__init__.py +29 -0
- package/adl_drift/lang/java/detectors/additional.py +445 -0
- package/adl_drift/lang/java/detectors/base.py +154 -0
- package/adl_drift/lang/java/detectors/data_access.py +148 -0
- package/adl_drift/lang/java/detectors/imports.py +53 -0
- package/adl_drift/lang/java/detectors/routes.py +159 -0
- package/adl_drift/lang/java/detectors/security.py +386 -0
- package/adl_drift/lang/java/detectors/tests.py +74 -0
- package/adl_drift/lang/java/extractor.py +582 -0
- package/adl_drift/lang/java/ids.py +99 -0
- package/adl_drift/lang/java/parser.py +49 -0
- package/adl_drift/lang/java/resolution/__init__.py +1 -0
- package/adl_drift/lang/java/resolution/constants.py +1619 -0
- package/adl_drift/lang/java/resolution/constructors.py +278 -0
- package/adl_drift/lang/java/resolution/context.py +305 -0
- package/adl_drift/lang/java/resolution/dispatch.py +540 -0
- package/adl_drift/lang/java/resolution/functional.py +1318 -0
- package/adl_drift/lang/java/resolution/generated.py +95 -0
- package/adl_drift/lang/java/resolution/model.py +153 -0
- package/adl_drift/lang/java/resolution/orchestration.py +866 -0
- package/adl_drift/lang/java/resolution/overloads.py +1275 -0
- package/adl_drift/lang/java/resolution/scopes.py +1111 -0
- package/adl_drift/lang/java/resolution/types.py +2947 -0
- package/adl_drift/lang/java/resolver.py +1401 -0
- package/adl_drift/lang/java/scopes.py +69 -0
- package/adl_drift/lang/java/taint.py +500 -0
- package/adl_drift/lang/java/type_provenance.py +342 -0
- package/adl_drift/lang/python/__init__.py +8 -0
- package/adl_drift/lang/python/backend.py +94 -0
- package/adl_drift/lang/python/cfg.py +537 -0
- package/adl_drift/lang/python/extractor.py +454 -0
- package/adl_drift/lang/python/parser.py +41 -0
- package/adl_drift/lang/python/resolver.py +1789 -0
- package/adl_drift/lang/python/taint.py +365 -0
- package/adl_drift/lang/registry.py +117 -0
- package/adl_drift/parser/__init__.py +1 -0
- package/adl_drift/parser/cache.py +38 -0
- package/adl_drift/parser/core.py +33 -0
- package/adl_drift/parser/extractor.py +17 -0
- package/adl_drift/parser/literal_eval.py +326 -0
- package/adl_drift/scanner/__init__.py +1 -0
- package/adl_drift/scanner/discovery.py +338 -0
- package/adl_drift/scanner/incremental.py +261 -0
- package/adl_drift/scanner/parallel.py +224 -0
- package/adl_drift/server/__init__.py +1 -0
- package/adl_drift/server/handlers.py +2849 -0
- package/adl_drift/server/mcp_server.py +127 -0
- package/adl_drift/server/mcp_stdio.py +335 -0
- package/adl_drift/server/protocol.py +71 -0
- package/adl_drift/server/static/index.html +696 -0
- package/adl_drift/server/web.py +350 -0
- package/adl_drift/utils/__init__.py +1 -0
- package/adl_drift/utils/config.py +77 -0
- package/adl_drift/utils/fswalk.py +52 -0
- package/adl_drift/utils/types.py +345 -0
- package/bin/adl-drift.mjs +35 -0
- package/bin/adl-toolkit.mjs +488 -0
- package/package.json +29 -4
- package/pyproject.toml +49 -0
- package/scripts/install_toolkit.py +3584 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# ADL Drift — predict-edit-prove workflow
|
|
2
|
+
|
|
3
|
+
ADL Drift indexes Python and Java symbols as a persistent call graph exposed through the `adl-drift` MCP server.
|
|
4
|
+
|
|
5
|
+
If there is no `.adl-drift/` directory, skip ADL-Drift entirely — indexing is the user's decision. The workflow below applies only when `.adl-drift/` exists at the project root.
|
|
6
|
+
|
|
7
|
+
1. **Explore read-only questions:** For a symbol's source/details, callers/blast radius, or dependencies, use `drift_explore` before text search or file reading. “What breaks if I delete/change X?” and refactor-safety questions also use `drift_explore`.
|
|
8
|
+
2. **Predict edits:** Before the first edit to a Python/Java function, method, constructor, class, field, constant, or parameter, **MUST call `drift_plan_change`**. It resolves the target itself, so `drift_explore` is not required first. For an addition, plan against the containing or related symbol. Save the returned `baseline`.
|
|
9
|
+
|
|
10
|
+
Example: `Add required audit_tag parameter to Python function jsonable_encoder`.
|
|
11
|
+
3. **Edit:** Update the target and affected callers, implementations, entry points, and tests.
|
|
12
|
+
4. **Prove:** After Python/Java edits, **MUST call `drift_verify_change`** with the original `baseline` and all modified Python/Java files in `changed_files`. Fix unintended differences and rerun with the same baseline; report intentional differences or a remaining `INCONSISTENT` verdict accurately.
|
|
13
|
+
5. **Validate:** Run relevant tests and static checks. If checks require further edits, repeat verification and affected checks. Graph consistency does not prove behavioral correctness.
|
|
14
|
+
|
|
15
|
+
If a target is uncertain, use `drift_explore` to inspect the resolved and ambiguous matches; use `drift_search` only when an exact qualified symbol still needs confirmation. Include parameter types for Java overloads. Treat fuzzy matches and low-confidence/framework-dispatched edges as leads to inspect.
|
|
16
|
+
|
|
17
|
+
Before the first graph query in a task, call `drift_status`. If the existing index is stale or incomplete, run `drift_scan` on the project root before exploring or planning. Recheck with `drift_status` after external source changes; after your own edits, use `drift_verify_change` with the original baseline. Inspect or report `scan_errors`. Increase `max_tokens` moderately for truncated results.
|
|
18
|
+
|
|
19
|
+
Use text search for literals, comments, logs, generated files, and files outside Drift's `.py`/`.java` coverage.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# ADL Drift — predict-edit-prove workflow
|
|
2
|
+
|
|
3
|
+
ADL Drift indexes Python and Java symbols as a persistent call graph exposed through the `adl-drift` MCP server.
|
|
4
|
+
|
|
5
|
+
If there is no `.adl-drift/` directory, skip ADL-Drift entirely — indexing is the user's decision. The workflow below applies only when `.adl-drift/` exists at the project root.
|
|
6
|
+
|
|
7
|
+
1. **Explore read-only questions:** For a symbol's source/details, callers/blast radius, or dependencies, use `drift_explore` before text search or file reading. “What breaks if I delete/change X?” and refactor-safety questions also use `drift_explore`.
|
|
8
|
+
2. **Predict edits:** Before the first edit to a Python/Java function, method, constructor, class, field, constant, or parameter, **MUST call `drift_plan_change`**. It resolves the target itself, so `drift_explore` is not required first. For an addition, plan against the containing or related symbol. Save the returned `baseline`.
|
|
9
|
+
|
|
10
|
+
Example: `Add required audit_tag parameter to Python function jsonable_encoder`.
|
|
11
|
+
3. **Edit:** Update the target and affected callers, implementations, entry points, and tests.
|
|
12
|
+
4. **Prove:** After Python/Java edits, **MUST call `drift_verify_change`** with the original `baseline` and all modified Python/Java files in `changed_files`. Fix unintended differences and rerun with the same baseline; report intentional differences or a remaining `INCONSISTENT` verdict accurately.
|
|
13
|
+
5. **Validate:** Run relevant tests and static checks. If checks require further edits, repeat verification and affected checks. Graph consistency does not prove behavioral correctness.
|
|
14
|
+
|
|
15
|
+
If a target is uncertain, use `drift_explore` to inspect the resolved and ambiguous matches; use `drift_search` only when an exact qualified symbol still needs confirmation. Include parameter types for Java overloads. Treat fuzzy matches and low-confidence/framework-dispatched edges as leads to inspect.
|
|
16
|
+
|
|
17
|
+
Before the first graph query in a task, call `drift_status`. If the existing index is stale or incomplete, run `drift_scan` on the project root before exploring or planning. Recheck with `drift_status` after external source changes; after your own edits, use `drift_verify_change` with the original baseline. Inspect or report `scan_errors`. Increase `max_tokens` moderately for truncated results.
|
|
18
|
+
|
|
19
|
+
Use text search for literals, comments, logs, generated files, and files outside Drift's `.py`/`.java` coverage.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: adl-drift
|
|
3
|
+
description: Persistent Python/Java call graph for symbol details, dependencies,
|
|
4
|
+
blast radius, and mandatory predict-edit-prove checks around source changes.
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# ADL Drift — predict-edit-prove workflow
|
|
8
|
+
|
|
9
|
+
ADL Drift indexes Python and Java symbols as a persistent call graph exposed through the `adl-drift` MCP server.
|
|
10
|
+
|
|
11
|
+
If there is no `.adl-drift/` directory, skip ADL-Drift entirely — indexing is the user's decision. The workflow below applies only when `.adl-drift/` exists at the project root.
|
|
12
|
+
|
|
13
|
+
1. **Explore read-only questions:** For a symbol's source/details, callers/blast radius, or dependencies, use `drift_explore` before text search or file reading. “What breaks if I delete/change X?” and refactor-safety questions also use `drift_explore`.
|
|
14
|
+
2. **Predict edits:** Before the first edit to a Python/Java function, method, constructor, class, field, constant, or parameter, **MUST call `drift_plan_change`**. It resolves the target itself, so `drift_explore` is not required first. For an addition, plan against the containing or related symbol. Save the returned `baseline`.
|
|
15
|
+
|
|
16
|
+
Example: `Add required audit_tag parameter to Python function jsonable_encoder`.
|
|
17
|
+
3. **Edit:** Update the target and affected callers, implementations, entry points, and tests.
|
|
18
|
+
4. **Prove:** After Python/Java edits, **MUST call `drift_verify_change`** with the original `baseline` and all modified Python/Java files in `changed_files`. Fix unintended differences and rerun with the same baseline; report intentional differences or a remaining `INCONSISTENT` verdict accurately.
|
|
19
|
+
5. **Validate:** Run relevant tests and static checks. If checks require further edits, repeat verification and affected checks. Graph consistency does not prove behavioral correctness.
|
|
20
|
+
|
|
21
|
+
If a target is uncertain, use `drift_explore` to inspect the resolved and ambiguous matches; use `drift_search` only when an exact qualified symbol still needs confirmation. Include parameter types for Java overloads. Treat fuzzy matches and low-confidence/framework-dispatched edges as leads to inspect.
|
|
22
|
+
|
|
23
|
+
Before the first graph query in a task, call `drift_status`. If the existing index is stale or incomplete, run `drift_scan` on the project root before exploring or planning. Recheck with `drift_status` after external source changes; after your own edits, use `drift_verify_change` with the original baseline. Inspect or report `scan_errors`. Increase `max_tokens` moderately for truncated results.
|
|
24
|
+
|
|
25
|
+
Use text search for literals, comments, logs, generated files, and files outside Drift's `.py`/`.java` coverage.
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""`adl-drift watch` — keep the index fresh while files are being edited.
|
|
2
|
+
|
|
3
|
+
A polling loop, not an OS watcher: the cheap pass stats mtimes of every
|
|
4
|
+
tracked backend source file; only when something moved does it run an incremental
|
|
5
|
+
scan, where `IncrementalScanner`'s content-hash classification filters
|
|
6
|
+
spurious mtime bumps (touch/checkout without content change). This keeps
|
|
7
|
+
the dependency surface at zero — the same trade web.py makes for the viz.
|
|
8
|
+
|
|
9
|
+
The watcher is optional. `MCPHandlers._build_full_graph()` re-classifies
|
|
10
|
+
files per call, so agents never read a stale graph even without it; the
|
|
11
|
+
watcher just moves the rescan cost off the agent's first tool call.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import time
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Callable, Dict, Optional
|
|
19
|
+
|
|
20
|
+
from adl_drift.cli.commands import DriftCommands
|
|
21
|
+
|
|
22
|
+
Snapshot = Dict[Path, float]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _snapshot(commands: DriftCommands, language: Optional[str] = None) -> Snapshot:
|
|
26
|
+
snap: Snapshot = {}
|
|
27
|
+
for fp in commands._find_files(commands.project_path, language):
|
|
28
|
+
try:
|
|
29
|
+
snap[fp] = fp.stat().st_mtime
|
|
30
|
+
except OSError:
|
|
31
|
+
continue # deleted between listing and stat — next pass catches it
|
|
32
|
+
return snap
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def watch(
|
|
36
|
+
project_path: Path,
|
|
37
|
+
interval: float = 2.0,
|
|
38
|
+
settle: float = 0.5,
|
|
39
|
+
max_iterations: Optional[int] = None,
|
|
40
|
+
on_event: Optional[Callable[[str], None]] = None,
|
|
41
|
+
language: Optional[str] = None,
|
|
42
|
+
) -> int:
|
|
43
|
+
"""Poll for changes and incrementally rescan. Blocks until interrupted
|
|
44
|
+
(or until *max_iterations* polls, for tests). Returns rescan count."""
|
|
45
|
+
project_path = Path(project_path).resolve()
|
|
46
|
+
commands = DriftCommands(project_path)
|
|
47
|
+
emit = on_event or (lambda msg: None)
|
|
48
|
+
|
|
49
|
+
emit(f"watching {project_path} (poll every {interval:.0f}s, Ctrl-C to stop)")
|
|
50
|
+
results = commands.scan(project_path, incremental=True, language=language)
|
|
51
|
+
# Snapshot BEFORE emitting so any listener-triggered write is not captured
|
|
52
|
+
# in `last` (which would hide the subsequent change from the poll loop).
|
|
53
|
+
last = _snapshot(commands, language)
|
|
54
|
+
emit(f"initial index: {len(results)} patterns, {len(commands.last_scan_errors)} errors")
|
|
55
|
+
rescans = 0
|
|
56
|
+
iterations = 0
|
|
57
|
+
|
|
58
|
+
while max_iterations is None or iterations < max_iterations:
|
|
59
|
+
iterations += 1
|
|
60
|
+
time.sleep(interval)
|
|
61
|
+
current = _snapshot(commands, language)
|
|
62
|
+
if current == last:
|
|
63
|
+
continue
|
|
64
|
+
time.sleep(settle) # let a save burst finish before rescanning
|
|
65
|
+
current = _snapshot(commands, language)
|
|
66
|
+
|
|
67
|
+
changed_paths = [
|
|
68
|
+
fp for fp in current
|
|
69
|
+
if last.get(fp) != current[fp]
|
|
70
|
+
] + [fp for fp in last if fp not in current]
|
|
71
|
+
start = time.monotonic()
|
|
72
|
+
results = commands.scan(project_path, incremental=True, language=language)
|
|
73
|
+
elapsed = time.monotonic() - start
|
|
74
|
+
rescans += 1
|
|
75
|
+
emit(
|
|
76
|
+
f"rescan #{rescans}: {len(changed_paths)} file(s) changed, "
|
|
77
|
+
f"{len(results)} patterns, {len(commands.last_scan_errors)} errors "
|
|
78
|
+
f"({elapsed:.1f}s)"
|
|
79
|
+
)
|
|
80
|
+
for err in commands.last_scan_errors:
|
|
81
|
+
emit(f" scan error: {err['file']} — {err['error']}")
|
|
82
|
+
last = current
|
|
83
|
+
|
|
84
|
+
return rescans
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Language-backend extension seam (P4.0, PROBLEMS.md).
|
|
2
|
+
|
|
3
|
+
`adl_drift.lang` is the boundary between the language-agnostic engine
|
|
4
|
+
(scanner, cli/commands, server/handlers, cortex) and language-coupled
|
|
5
|
+
analysis (parsing, symbol extraction, detectors, name resolution, CFG/taint,
|
|
6
|
+
framework detection). See `lang.base.LanguageBackend` for the contract and
|
|
7
|
+
`lang.registry` for backend dispatch.
|
|
8
|
+
"""
|
|
@@ -0,0 +1,464 @@
|
|
|
1
|
+
"""The `LanguageBackend` contract — single source of truth for P4.1-P4.11.
|
|
2
|
+
|
|
3
|
+
See PROBLEMS.md P4.0 for the full design writeup. This module defines the
|
|
4
|
+
seam a language plugs into; it does not implement any language (with one
|
|
5
|
+
exception: `LanguageParser`, the grammar-agnostic tree-sitter wrapper, is a
|
|
6
|
+
real class as of P4.1 — see its docstring). The rest of the Python pipeline
|
|
7
|
+
(extractor.py, detectors/, the resolver in graph/builder.py, graph/cfg.py,
|
|
8
|
+
detectors/taint.py, utils/config.py) is migrated behind this contract in
|
|
9
|
+
P4.2-P4.7. A Java backend plugs in the same way without forking the engine
|
|
10
|
+
layer (scanner, cli/commands, server/handlers, cortex) — those call sites ask
|
|
11
|
+
`lang.registry` for a backend, never import a concrete one.
|
|
12
|
+
|
|
13
|
+
Boundary rules (P4.0):
|
|
14
|
+
- The graph data model (`graph/node.py`, `graph/edges.py`, `EdgeType`, the
|
|
15
|
+
`module:qualname` node-id shape) stays shared and is untouched here.
|
|
16
|
+
- `Symbol` / `Pattern` / `AnalysisResult` (`utils/types.py`) stay shared
|
|
17
|
+
dataclasses; language-specific needs are additive fields (P4.9), not new
|
|
18
|
+
per-language types.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import time
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import (
|
|
27
|
+
TYPE_CHECKING,
|
|
28
|
+
Any,
|
|
29
|
+
Callable,
|
|
30
|
+
Dict,
|
|
31
|
+
FrozenSet,
|
|
32
|
+
List,
|
|
33
|
+
Optional,
|
|
34
|
+
Protocol,
|
|
35
|
+
Tuple,
|
|
36
|
+
runtime_checkable,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
from tree_sitter import Language, Node, Parser, Query, QueryCursor, Tree
|
|
40
|
+
|
|
41
|
+
from adl_drift.utils.types import AnalysisResult, ImportRecord, Symbol
|
|
42
|
+
|
|
43
|
+
if TYPE_CHECKING:
|
|
44
|
+
# `BaseDetector` is only used as a type annotation here. Keep the import
|
|
45
|
+
# type-checker-only so the language contract does not load detector plugins.
|
|
46
|
+
from adl_drift.detectors.base import BaseDetector
|
|
47
|
+
# Same story for `CallGraphBuilder`: `graph/builder.py` lazily imports
|
|
48
|
+
# concrete `CallResolver` subclasses *inside a method body* (P4.4), so
|
|
49
|
+
# there is no real runtime cycle — but importing the class at module
|
|
50
|
+
# scope here just to spell out a type hint would still be needless
|
|
51
|
+
# coupling for a module whose whole job is staying grammar/engine-agnostic.
|
|
52
|
+
from adl_drift.graph.builder import CallGraphBuilder
|
|
53
|
+
|
|
54
|
+
__all__ = [
|
|
55
|
+
"ParseResult",
|
|
56
|
+
"LanguageParser",
|
|
57
|
+
"SymbolExtractor",
|
|
58
|
+
"ResolverConfig",
|
|
59
|
+
"CallResolver",
|
|
60
|
+
"CFGConfig",
|
|
61
|
+
"ProjectConfig",
|
|
62
|
+
"LanguageBackend",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class ParseResult:
|
|
68
|
+
"""Output of a single parse. Already language-agnostic — unchanged by
|
|
69
|
+
P4.1 (Tree + source bytes + error nodes + parse time, whatever the
|
|
70
|
+
grammar)."""
|
|
71
|
+
|
|
72
|
+
tree: Tree
|
|
73
|
+
source_bytes: bytes
|
|
74
|
+
parse_time_ms: float = 0.0
|
|
75
|
+
errors: List[Node] = field(default_factory=list)
|
|
76
|
+
file_path: Optional[Path] = None
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def has_errors(self) -> bool:
|
|
80
|
+
return len(self.errors) > 0
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class LanguageParser:
|
|
84
|
+
"""Grammar-agnostic tree-sitter wrapper (P4.1, PROBLEMS.md).
|
|
85
|
+
|
|
86
|
+
Generalized out of `parser/core.py::PythonParser`: takes a
|
|
87
|
+
`tree_sitter.Language` and a query-name -> query-string map at
|
|
88
|
+
construction instead of relying on a module-global grammar. Error-node
|
|
89
|
+
collection, byte-offset -> `SourcePosition` conversion (`node_position`),
|
|
90
|
+
and the `node_text` helper are unchanged from the Python-only version.
|
|
91
|
+
|
|
92
|
+
`lang/python/parser.py` and `lang/java/parser.py` each own the grammar
|
|
93
|
+
load (`tree_sitter_python`/`tree_sitter_java`) and query map, and
|
|
94
|
+
construct one of these configured for their language.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
def __init__(self, language: Language, queries: Dict[str, str]) -> None:
|
|
98
|
+
self.language = language
|
|
99
|
+
self.queries = queries
|
|
100
|
+
self._parser = Parser(language)
|
|
101
|
+
|
|
102
|
+
def parse_string(self, source: str, file_path: Optional[Path] = None) -> ParseResult:
|
|
103
|
+
source_bytes = source.encode("utf-8")
|
|
104
|
+
start = time.perf_counter()
|
|
105
|
+
tree = self._parser.parse(source_bytes)
|
|
106
|
+
elapsed = (time.perf_counter() - start) * 1000.0
|
|
107
|
+
errors = self._collect_error_nodes(tree.root_node)
|
|
108
|
+
return ParseResult(
|
|
109
|
+
tree=tree,
|
|
110
|
+
source_bytes=source_bytes,
|
|
111
|
+
parse_time_ms=elapsed,
|
|
112
|
+
errors=errors,
|
|
113
|
+
file_path=file_path,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
def parse_file(self, path: Path) -> ParseResult:
|
|
117
|
+
path = Path(path)
|
|
118
|
+
source = path.read_text(encoding="utf-8", errors="replace")
|
|
119
|
+
return self.parse_string(source, file_path=path)
|
|
120
|
+
|
|
121
|
+
def query_nodes(self, tree: Tree, query_name: str) -> List[Node]:
|
|
122
|
+
if query_name not in self.queries:
|
|
123
|
+
raise KeyError(f"Unknown query: {query_name}")
|
|
124
|
+
query = Query(self.language, self.queries[query_name])
|
|
125
|
+
captures = QueryCursor(query).captures(tree.root_node)
|
|
126
|
+
if isinstance(captures, dict):
|
|
127
|
+
nodes: List[Node] = []
|
|
128
|
+
for vals in captures.values():
|
|
129
|
+
nodes.extend(vals)
|
|
130
|
+
return nodes
|
|
131
|
+
return [n for n, _ in captures]
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def node_text(node: Node, source: bytes) -> str:
|
|
135
|
+
return source[node.start_byte : node.end_byte].decode("utf-8", errors="replace")
|
|
136
|
+
|
|
137
|
+
@staticmethod
|
|
138
|
+
def node_position(node: Node):
|
|
139
|
+
from adl_drift.utils.types import SourcePosition
|
|
140
|
+
|
|
141
|
+
sr, sc = node.start_point
|
|
142
|
+
er, ec = node.end_point
|
|
143
|
+
return SourcePosition(line=sr + 1, column=sc, end_line=er + 1, end_column=ec)
|
|
144
|
+
|
|
145
|
+
@classmethod
|
|
146
|
+
def _collect_error_nodes(cls, node: Node) -> List[Node]:
|
|
147
|
+
out: List[Node] = []
|
|
148
|
+
if node.type == "ERROR" or node.is_missing:
|
|
149
|
+
out.append(node)
|
|
150
|
+
for child in node.children:
|
|
151
|
+
out.extend(cls._collect_error_nodes(child))
|
|
152
|
+
return out
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class SymbolExtractor:
|
|
156
|
+
"""CST -> `AnalysisResult` (P4.2, PROBLEMS.md).
|
|
157
|
+
|
|
158
|
+
Promoted from a placeholder `Protocol` to a real base class: owns the
|
|
159
|
+
parts of symbol extraction that do not depend on any one grammar —
|
|
160
|
+
`extract()`/`extract_string()` (parse then delegate to `_build()`), the
|
|
161
|
+
depth-first CST walk (`_walk`), the dedupe-by-`(scope_chain, parent,
|
|
162
|
+
name)` key used to drop duplicate variable/attribute symbols
|
|
163
|
+
(`_dedupe_key`), and final `AnalysisResult` assembly including the
|
|
164
|
+
`ImportRecord`/`imports` plumbing and parse-error formatting
|
|
165
|
+
(`_finalize`). `_build()` itself — the node-type dispatch table — is
|
|
166
|
+
grammar-specific and left abstract here; `lang/python/extractor.py` and
|
|
167
|
+
`lang/java/extractor.py` each implement it against their own CST node
|
|
168
|
+
types.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
def __init__(self, parser: "LanguageParser") -> None:
|
|
172
|
+
self.parser = parser
|
|
173
|
+
|
|
174
|
+
def extract(self, file_path: Path) -> AnalysisResult:
|
|
175
|
+
parse = self.parser.parse_file(file_path)
|
|
176
|
+
return self._build(parse, file_path)
|
|
177
|
+
|
|
178
|
+
def extract_string(
|
|
179
|
+
self, source: str, file_path: Optional[Path] = None
|
|
180
|
+
) -> AnalysisResult:
|
|
181
|
+
parse = self.parser.parse_string(source, file_path=file_path)
|
|
182
|
+
return self._build(parse, file_path or Path("<string>"))
|
|
183
|
+
|
|
184
|
+
def _build(self, parse: ParseResult, file_path: Path) -> AnalysisResult:
|
|
185
|
+
"""Grammar-specific CST walk -> `AnalysisResult`. Implemented per language."""
|
|
186
|
+
raise NotImplementedError
|
|
187
|
+
|
|
188
|
+
def _walk(self, node: Node):
|
|
189
|
+
yield node
|
|
190
|
+
for child in node.children:
|
|
191
|
+
yield from self._walk(child)
|
|
192
|
+
|
|
193
|
+
@staticmethod
|
|
194
|
+
def _dedupe_key(symbol: Symbol) -> Tuple[Tuple[str, ...], str, str]:
|
|
195
|
+
"""Uniqueness key for symbols that may be visited more than once
|
|
196
|
+
(e.g. variable/attribute-style declarations). Not applied to
|
|
197
|
+
function/method symbols — legitimate Java overloads share this key
|
|
198
|
+
and must stay distinct (P4.5's `signature_key` is what disambiguates
|
|
199
|
+
them, not this dedupe pass)."""
|
|
200
|
+
return (tuple(symbol.scope_chain), symbol.parent or "", symbol.name)
|
|
201
|
+
|
|
202
|
+
@staticmethod
|
|
203
|
+
def _finalize(
|
|
204
|
+
parse: ParseResult,
|
|
205
|
+
file_path: Path,
|
|
206
|
+
symbols: List[Symbol],
|
|
207
|
+
imports: List[str],
|
|
208
|
+
import_records: List[ImportRecord],
|
|
209
|
+
*,
|
|
210
|
+
language: str = "python",
|
|
211
|
+
module_name: Optional[str] = None,
|
|
212
|
+
) -> AnalysisResult:
|
|
213
|
+
return AnalysisResult(
|
|
214
|
+
file_path=file_path,
|
|
215
|
+
symbols=symbols,
|
|
216
|
+
imports=imports,
|
|
217
|
+
import_records=import_records,
|
|
218
|
+
errors=[f"parse error at {n.start_point}" for n in parse.errors],
|
|
219
|
+
parse_time_ms=parse.parse_time_ms,
|
|
220
|
+
language=language,
|
|
221
|
+
module_name=module_name,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@dataclass
|
|
226
|
+
class ResolverConfig:
|
|
227
|
+
"""Name-resolution *policy* for the language-agnostic call-resolution
|
|
228
|
+
seam (P4.4, PROBLEMS.md).
|
|
229
|
+
|
|
230
|
+
`LanguageBackend.resolver_config()` returns one of these.
|
|
231
|
+
`CallGraphBuilder._get_resolver()` uses `resolver_factory` to build the
|
|
232
|
+
`CallResolver` instance that actually walks this language's CST and
|
|
233
|
+
applies the policy below. Splitting declarative policy (this dataclass)
|
|
234
|
+
from CST-coupled mechanics (`CallResolver`) mirrors how P4.2 split
|
|
235
|
+
`SymbolExtractor._build()` (grammar-specific) from the shared parts of
|
|
236
|
+
that base class. Graph mechanics that stay language-agnostic — edge
|
|
237
|
+
dedupe (`CallGraphBuilder._add_edge`), the MultiDiGraph, confidence
|
|
238
|
+
persistence, impact BFS, and the id-resolution helpers
|
|
239
|
+
(`_resolve_name`/`_contains_child`/`_find_method_on_ancestors`/
|
|
240
|
+
`_resolve_annotation`, which walk CONTAINS/INHERITS edges rather than
|
|
241
|
+
any grammar) are *not* part of this config; they live on
|
|
242
|
+
`CallGraphBuilder` and are reused unchanged by every language.
|
|
243
|
+
|
|
244
|
+
- `receiver_self_names`: identifiers meaning "the enclosing instance"
|
|
245
|
+
(Python: ``("self", "cls")``; Java: ``("this",)``).
|
|
246
|
+
- `receiver_super_names`: identifiers meaning "walk the ancestor chain"
|
|
247
|
+
(Python: ``("super",)``; Java: ``("super",)``).
|
|
248
|
+
- `builtins_deny_list`: names the fuzzy / over-approximation strategies
|
|
249
|
+
must never resolve through (P2.7's `PYTHON_BUILTINS`; Java's
|
|
250
|
+
`java.lang.Object` methods + common collection methods).
|
|
251
|
+
- `annotation_stub_table`: decorator/annotation simple name -> the set
|
|
252
|
+
of names it contributes, read differently per language
|
|
253
|
+
(`DECORATOR_STUB_TABLE`: exact attribute names a Python decorator adds
|
|
254
|
+
to the wrapped callable; `JAVA_ANNOTATION_STUB_TABLE`: name *prefixes*
|
|
255
|
+
Lombok-style annotations generate on the annotated class, since those
|
|
256
|
+
names are field-dependent and not a fixed set).
|
|
257
|
+
- `resolver_factory`: builds the `CallResolver` for this policy. Kept on
|
|
258
|
+
the config rather than a new `LanguageBackend` method so P4.0's
|
|
259
|
+
`LanguageBackend` Protocol surface stays exactly as originally specified.
|
|
260
|
+
"""
|
|
261
|
+
|
|
262
|
+
receiver_self_names: Tuple[str, ...] = ()
|
|
263
|
+
receiver_super_names: Tuple[str, ...] = ()
|
|
264
|
+
builtins_deny_list: FrozenSet[str] = frozenset()
|
|
265
|
+
annotation_stub_table: Dict[str, FrozenSet[str]] = field(default_factory=dict)
|
|
266
|
+
resolver_factory: Optional[
|
|
267
|
+
Callable[["CallGraphBuilder", "ResolverConfig"], "CallResolver"]
|
|
268
|
+
] = None
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class CallResolver:
|
|
272
|
+
"""Per-language call-resolution strategy (P4.4, PROBLEMS.md).
|
|
273
|
+
|
|
274
|
+
Owns everything CST-coupled about turning a function/method body into
|
|
275
|
+
CALLS/INSTANTIATES/AWAITS/REFERENCES edges for one language: what a call
|
|
276
|
+
site looks like in this grammar, how a bare/qualified receiver resolves
|
|
277
|
+
to a name, and (Java) overload/erasure handling. Grammar-agnostic id
|
|
278
|
+
resolution over the graph itself (`_resolve_name`, `_contains_child`,
|
|
279
|
+
`_find_method_on_ancestors`, `_resolve_annotation`, edge dedupe) stays on
|
|
280
|
+
`CallGraphBuilder` and is reached via `self.builder`.
|
|
281
|
+
|
|
282
|
+
`lang/python/resolver.py::PythonCallResolver` and
|
|
283
|
+
`lang/java/resolver.py::JavaCallResolver` are the two implementations.
|
|
284
|
+
Never constructed directly — `CallGraphBuilder._get_resolver()` builds
|
|
285
|
+
one lazily via `ResolverConfig.resolver_factory`.
|
|
286
|
+
"""
|
|
287
|
+
|
|
288
|
+
def __init__(self, builder: "CallGraphBuilder", config: ResolverConfig) -> None:
|
|
289
|
+
self.builder = builder
|
|
290
|
+
self.config = config
|
|
291
|
+
|
|
292
|
+
def resolve_calls(
|
|
293
|
+
self,
|
|
294
|
+
parses: Dict[Path, "ParseResult"],
|
|
295
|
+
results: Dict[Path, AnalysisResult],
|
|
296
|
+
) -> int:
|
|
297
|
+
"""Emit CALLS/INSTANTIATES/AWAITS/REFERENCES edges for every call
|
|
298
|
+
site in `parses`. Returns the number of CALLS/INSTANTIATES edges
|
|
299
|
+
added."""
|
|
300
|
+
raise NotImplementedError
|
|
301
|
+
|
|
302
|
+
def resolve_decorator_types(
|
|
303
|
+
self,
|
|
304
|
+
parses: Dict[Path, "ParseResult"],
|
|
305
|
+
results: Dict[Path, AnalysisResult],
|
|
306
|
+
) -> None:
|
|
307
|
+
"""Optional pre-pass building decorator/annotation-stub bindings
|
|
308
|
+
consulted by `resolve_calls()`. No-op unless a language needs one
|
|
309
|
+
(Python's P2.15 stub table + narrow own-inference; Java's simpler
|
|
310
|
+
annotation-stub lookup resolves inline and needs no pre-pass)."""
|
|
311
|
+
return None
|
|
312
|
+
|
|
313
|
+
def harvest_literal_candidates(
|
|
314
|
+
self,
|
|
315
|
+
parses: Dict[Path, "ParseResult"],
|
|
316
|
+
results: Dict[Path, AnalysisResult],
|
|
317
|
+
) -> int:
|
|
318
|
+
"""Optional low-confidence candidate-edge pass over literal/dynamic
|
|
319
|
+
dispatch idioms (Python's P2.13/P2.22/P2.23 dict-router patterns).
|
|
320
|
+
No-op by default; returns edges added."""
|
|
321
|
+
return 0
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
@dataclass
|
|
325
|
+
class CFGConfig:
|
|
326
|
+
"""Statement-kind -> CFG-effect mapping + taint source/sink tables (P4.6).
|
|
327
|
+
|
|
328
|
+
`LanguageBackend.cfg_config()` returns one of these. It is consumed by
|
|
329
|
+
two generic engines that stay language-agnostic (P4.6, PROBLEMS.md):
|
|
330
|
+
`graph/cfg.py::CFGBuilderBase`/`ReachingDefinitions` (the GEN/KILL
|
|
331
|
+
fixpoint + `DefUseFact` emission) and `detectors/taint.py::TaintAnalyzer`
|
|
332
|
+
(the cross-function taint worklist + `TaintPath` dataclass +
|
|
333
|
+
`taint_paths` persistence). Neither of those engines is CST-coupled —
|
|
334
|
+
they operate on `CFGBlock`/`CFGDef` graphs and `(tainted, TaintPath,
|
|
335
|
+
_OutCall)` triples respectively, never on raw tree-sitter node types —
|
|
336
|
+
so, mirroring how P4.4 split `ResolverConfig` (declarative policy) from
|
|
337
|
+
`CallResolver` (CST-coupled mechanics), this dataclass carries only
|
|
338
|
+
*data* (the tables below) plus two factories that build the CST-coupled
|
|
339
|
+
mechanics classes: `cfg_builder_factory` (a fresh
|
|
340
|
+
`graph.cfg.CFGBuilderBase` subclass instance per build — Python's is
|
|
341
|
+
`lang/python/cfg.py::PythonCFGBuilder`, Java's is
|
|
342
|
+
`lang/java/cfg.py::JavaCFGBuilder`) and `taint_strategy_factory` (a
|
|
343
|
+
`detectors.taint.TaintStrategy` subclass instance — `PythonTaintStrategy`
|
|
344
|
+
/ `JavaTaintStrategy`, living next to their respective `CFGBuilder`).
|
|
345
|
+
|
|
346
|
+
Every table defaults to empty so an unset field is simply "this language
|
|
347
|
+
has no such source/sink shape" rather than raising — e.g. Python has no
|
|
348
|
+
constructor-based sink (`taint_sink_constructors` stays `{}`) and no
|
|
349
|
+
parameter-annotation source (`taint_source_param_annotations` stays
|
|
350
|
+
`{}`), while Java populates both and leaves the legacy `taint_sources`
|
|
351
|
+
prefix-list empty (its sources are receiver-agnostic method names /
|
|
352
|
+
annotations, not RHS-text prefixes).
|
|
353
|
+
"""
|
|
354
|
+
|
|
355
|
+
# ---- taint sources -----------------------------------------------
|
|
356
|
+
taint_sources: Tuple[str, ...] = ()
|
|
357
|
+
"""Prefix-matched against a def/argument's *full* RHS text — Python's
|
|
358
|
+
original P2.18/P2.19 mechanism (`request.GET`, `os.environ.get`, ...)."""
|
|
359
|
+
|
|
360
|
+
taint_source_methods: Dict[str, str] = field(default_factory=dict)
|
|
361
|
+
"""Final call-target identifier (any receiver) -> source label. Java's
|
|
362
|
+
`getParameter`/`getHeader`/`getQueryString` are receiver-name-agnostic
|
|
363
|
+
(an `HttpServletRequest` variable can be called anything), unlike
|
|
364
|
+
Python's conventionally-named `request` object. Empty for Python."""
|
|
365
|
+
|
|
366
|
+
taint_source_param_annotations: Dict[str, str] = field(default_factory=dict)
|
|
367
|
+
"""Annotation simple name on a formal parameter -> source label. Java's
|
|
368
|
+
`@RequestParam`/`@PathVariable`/`@RequestBody`/JAX-RS `@QueryParam`/
|
|
369
|
+
`@PathParam` mark a parameter as tainted at function entry regardless of
|
|
370
|
+
caller (unlike an ordinary parameter, which only becomes tainted via
|
|
371
|
+
cross-function propagation). Empty for Python (no parameter-annotation
|
|
372
|
+
sources)."""
|
|
373
|
+
|
|
374
|
+
# ---- taint sinks ----------------------------------------------------
|
|
375
|
+
taint_sinks: Dict[str, str] = field(default_factory=dict)
|
|
376
|
+
"""Exact dotted-callable text -> vulnerability category. Python's
|
|
377
|
+
`eval`/`os.system`/`pickle.load`; Java's `System.getenv`-shaped exact
|
|
378
|
+
calls have no sink equivalent today, so this stays receiver-qualified
|
|
379
|
+
calls only (e.g. none needed beyond the method-name table below for the
|
|
380
|
+
current Java sink set)."""
|
|
381
|
+
|
|
382
|
+
sink_methods: Dict[str, str] = field(default_factory=dict)
|
|
383
|
+
"""Final call-target identifier (any receiver) -> category. Python's
|
|
384
|
+
`execute`/`executemany`/`raw`; Java's `executeQuery`/`executeUpdate`/
|
|
385
|
+
`createNativeQuery`/`readObject`/`exec`/`eval`/`invoke`/`forName`."""
|
|
386
|
+
|
|
387
|
+
sink_constructors: Dict[str, str] = field(default_factory=dict)
|
|
388
|
+
"""Constructed type simple name (`new X(...)`) -> category. Java's
|
|
389
|
+
`ProcessBuilder`. Empty for Python (no constructor-based sink)."""
|
|
390
|
+
|
|
391
|
+
# ---- CST-coupled mechanics factories --------------------------------
|
|
392
|
+
cfg_builder_factory: Optional[Callable[["CFGConfig"], "CFGBuilderBase"]] = None
|
|
393
|
+
"""Builds this language's `graph.cfg.CFGBuilderBase` subclass instance.
|
|
394
|
+
Mirrors `ResolverConfig.resolver_factory`."""
|
|
395
|
+
|
|
396
|
+
taint_strategy_factory: Optional[Callable[["CFGConfig"], "TaintStrategy"]] = None
|
|
397
|
+
"""Builds this language's `detectors.taint.TaintStrategy` subclass
|
|
398
|
+
instance."""
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
@runtime_checkable
|
|
402
|
+
class ProjectConfig(Protocol):
|
|
403
|
+
"""Framework auto-detection result for a project root.
|
|
404
|
+
|
|
405
|
+
Placeholder shape only, matched structurally to the existing
|
|
406
|
+
`utils.config.ProjectConfig` (Python-only today) so it satisfies this
|
|
407
|
+
Protocol without modification. A Java equivalent parsing `pom.xml` /
|
|
408
|
+
`build.gradle` lands in P4.7.
|
|
409
|
+
"""
|
|
410
|
+
|
|
411
|
+
project_path: Path
|
|
412
|
+
frameworks: List[str]
|
|
413
|
+
|
|
414
|
+
def has_framework(self, name: str) -> bool: ...
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
@runtime_checkable
|
|
418
|
+
class LanguageBackend(Protocol):
|
|
419
|
+
"""The extension seam every P4 sub-issue plugs into.
|
|
420
|
+
|
|
421
|
+
An implementation owns everything language-coupled for one language:
|
|
422
|
+
parsing, symbol extraction, detectors, name-resolution policy, CFG/taint
|
|
423
|
+
tables, framework detection, and module-id derivation. The engine layer
|
|
424
|
+
(scanner, cli/commands, server/handlers, cortex) is expected to obtain a
|
|
425
|
+
backend from `lang.registry` and never import a concrete implementation.
|
|
426
|
+
"""
|
|
427
|
+
|
|
428
|
+
name: str
|
|
429
|
+
"""Backend identifier, e.g. "python" | "java"."""
|
|
430
|
+
|
|
431
|
+
file_extensions: Tuple[str, ...]
|
|
432
|
+
"""Source file suffixes this backend claims, e.g. (".py",) | (".java",)."""
|
|
433
|
+
|
|
434
|
+
project_markers: Tuple[str, ...]
|
|
435
|
+
"""Files whose presence in a project root signals this language, e.g.
|
|
436
|
+
("pyproject.toml", "setup.py") | ("pom.xml", "build.gradle")."""
|
|
437
|
+
|
|
438
|
+
def parser(self) -> LanguageParser:
|
|
439
|
+
"""P4.1 — tree-sitter `Language` + query map for this grammar."""
|
|
440
|
+
...
|
|
441
|
+
|
|
442
|
+
def extractor(self) -> SymbolExtractor:
|
|
443
|
+
"""P4.2 — CST -> `AnalysisResult` symbol walk."""
|
|
444
|
+
...
|
|
445
|
+
|
|
446
|
+
def detectors(self) -> List[BaseDetector]:
|
|
447
|
+
"""P4.3 — this language's detector set."""
|
|
448
|
+
...
|
|
449
|
+
|
|
450
|
+
def resolver_config(self) -> ResolverConfig:
|
|
451
|
+
"""P4.4 — name-resolution rules + deny-list."""
|
|
452
|
+
...
|
|
453
|
+
|
|
454
|
+
def cfg_config(self) -> CFGConfig:
|
|
455
|
+
"""P4.6 — CFG node kinds + taint sources/sinks."""
|
|
456
|
+
...
|
|
457
|
+
|
|
458
|
+
def project_config(self, root: Path) -> ProjectConfig:
|
|
459
|
+
"""P4.7 — framework detection for the project rooted at `root`."""
|
|
460
|
+
...
|
|
461
|
+
|
|
462
|
+
def module_name(self, file: Path, root: Path) -> str:
|
|
463
|
+
"""P4.5 — derive the `module:qualname` module id for `file`."""
|
|
464
|
+
...
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Landing spot for the Java `LanguageBackend` (P4.1, PROBLEMS.md).
|
|
2
|
+
|
|
3
|
+
Implemented pieces: parser (P4.1), symbol extraction (P4.2), detectors (P4.3),
|
|
4
|
+
package/overload identity (P4.5), shared model support (P4.9), and Maven/Gradle
|
|
5
|
+
project configuration (P4.7). Resolution and CFG/taint land later.
|
|
6
|
+
"""
|