mlai-code 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/checkpoints.mjs +217 -0
- package/src/cli.mjs +153 -26
- package/src/cloud.mjs +2 -0
- package/src/engine.mjs +1 -1
- package/src/headless.mjs +39 -11
- package/src/home.mjs +3 -2
- package/src/host.mjs +141 -13
- package/src/ide-quick.mjs +147 -0
- package/src/ide.mjs +182 -0
- package/src/netbox.mjs +1 -1
- package/src/parity.mjs +17 -0
- package/src/relay.mjs +1 -1
- package/src/render.mjs +16 -5
- package/src/runtime.mjs +2 -0
- package/src/server.mjs +46 -7
- package/src/session.mjs +31 -8
- package/src/tui/acp.mjs +121 -19
- package/src/tui/app.mjs +2 -1
- package/src/tui/chat.mjs +35 -10
- package/src/tui/cmd-config.mjs +3 -1
- package/src/tui/cmd-core.mjs +9 -5
- package/src/tui/cmd-extensibility.mjs +5 -0
- package/src/tui/extensions-local.mjs +2 -0
- package/src/tui/mcp-config.mjs +22 -4
- package/src/tui/review-cli.mjs +15 -4
- package/src/tui/snapshots.mjs +57 -16
- package/src/tui/transcript.mjs +13 -2
- package/src/two-words.mjs +14 -8
- package/src/view-sync.mjs +48 -0
- package/src/winenv.mjs +1 -1
- package/src/workspace-api.mjs +21 -4
- package/vendor/client-sdk/index.cjs +2 -1
- package/vendor/cloud-cli/mlai.mjs +23 -11
- package/worker/architecture_worker.py +33 -4
- package/worker/browser_tool.py +19 -1
- package/worker/chat_fork.py +4 -1
- package/worker/checkpoints.py +71 -8
- package/worker/command_safety.py +56 -3
- package/worker/connected_turn.py +30 -0
- package/worker/conversation.py +125 -13
- package/worker/git_sync.py +4 -4
- package/worker/image_generation.py +2 -2
- package/worker/kimi_bridge.py +251 -31
- package/worker/kimi_gates.py +25 -6
- package/worker/kimi_hook.py +154 -13
- package/worker/media_tools.py +4 -1
- package/worker/mlai_hooks.py +222 -3
- package/worker/mlai_mcp.py +26 -6
- package/worker/mlai_review/README.md +35 -0
- package/worker/mlai_review/__init__.py +1 -0
- package/worker/mlai_review/__main__.py +261 -0
- package/worker/mlai_review/_compat.py +29 -0
- package/worker/mlai_review/analysis/__init__.py +3 -0
- package/worker/mlai_review/analysis/detect.py +50 -0
- package/worker/mlai_review/analysis/main.py +128 -0
- package/worker/mlai_review/analysis/messages.py +8 -0
- package/worker/mlai_review/analysis/tools.py +194 -0
- package/worker/mlai_review/application/__init__.py +1 -0
- package/worker/mlai_review/application/asking.py +53 -0
- package/worker/mlai_review/application/challengers.py +142 -0
- package/worker/mlai_review/application/context_builder.py +194 -0
- package/worker/mlai_review/application/errors.py +17 -0
- package/worker/mlai_review/application/feedback.py +52 -0
- package/worker/mlai_review/application/matching.py +46 -0
- package/worker/mlai_review/application/messages.py +41 -0
- package/worker/mlai_review/application/ports.py +222 -0
- package/worker/mlai_review/application/prompts.py +342 -0
- package/worker/mlai_review/application/review_service.py +200 -0
- package/worker/mlai_review/application/reviewers.py +438 -0
- package/worker/mlai_review/application/streaming.py +91 -0
- package/worker/mlai_review/application/tools.py +138 -0
- package/worker/mlai_review/application/usage.py +145 -0
- package/worker/mlai_review/application/write_check.py +46 -0
- package/worker/mlai_review/contracts/__init__.py +5 -0
- package/worker/mlai_review/contracts/analysis.py +57 -0
- package/worker/mlai_review/contracts/review.py +42 -0
- package/worker/mlai_review/domain/__init__.py +4 -0
- package/worker/mlai_review/domain/changes.py +77 -0
- package/worker/mlai_review/domain/context.py +147 -0
- package/worker/mlai_review/domain/diagnostics.py +239 -0
- package/worker/mlai_review/domain/feedback.py +216 -0
- package/worker/mlai_review/domain/findings.py +505 -0
- package/worker/mlai_review/domain/lifecycle.py +85 -0
- package/worker/mlai_review/domain/partition.py +42 -0
- package/worker/mlai_review/domain/relations.py +166 -0
- package/worker/mlai_review/domain/relevance.py +202 -0
- package/worker/mlai_review/domain/report.py +99 -0
- package/worker/mlai_review/domain/structure.py +56 -0
- package/worker/mlai_review/domain/writes.py +86 -0
- package/worker/mlai_review/eval.py +221 -0
- package/worker/mlai_review/eval_cases/clean-rename.json +11 -0
- package/worker/mlai_review/eval_cases/missing-ownership-check.json +22 -0
- package/worker/mlai_review/eval_cases/off-by-one.json +22 -0
- package/worker/mlai_review/eval_cases/sql-injection.json +20 -0
- package/worker/mlai_review/feedback_report.py +60 -0
- package/worker/mlai_review/infrastructure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/none.py +12 -0
- package/worker/mlai_review/infrastructure/analysis/translate.py +17 -0
- package/worker/mlai_review/infrastructure/github/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/github/client.py +445 -0
- package/worker/mlai_review/infrastructure/github/feedback.py +212 -0
- package/worker/mlai_review/infrastructure/github/http.py +31 -0
- package/worker/mlai_review/infrastructure/github/rendering.py +398 -0
- package/worker/mlai_review/infrastructure/github/token.py +20 -0
- package/worker/mlai_review/infrastructure/local/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/local/analyzer.py +48 -0
- package/worker/mlai_review/infrastructure/local/events.py +100 -0
- package/worker/mlai_review/infrastructure/local/git_reader.py +233 -0
- package/worker/mlai_review/infrastructure/local/publisher.py +138 -0
- package/worker/mlai_review/infrastructure/messages.py +54 -0
- package/worker/mlai_review/infrastructure/models/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/models/openai_compatible.py +269 -0
- package/worker/mlai_review/infrastructure/structure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/structure/heuristic.py +186 -0
- package/worker/mlai_review/infrastructure/structure/tree_sitter.py +186 -0
- package/worker/mlai_review/infrastructure/workspace/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/workspace/local.py +105 -0
- package/worker/mlai_review/infrastructure/workspace/memory.py +77 -0
- package/worker/mlai_review/infrastructure/workspace/paths.py +17 -0
- package/worker/mlai_review/pipeline.py +89 -0
- package/worker/mlai_review/runtime.py +194 -0
- package/worker/mlai_review/settings.py +139 -0
- package/worker/preview.py +15 -0
- package/worker/product_facts.py +3 -1
- package/worker/repo_policy.py +5 -0
- package/worker/roles.py +29 -2
- package/worker/runner.py +164 -23
- package/worker/runtime_control.py +95 -9
- package/worker/secret_scan.py +3 -2
- package/worker/server_memory.py +122 -0
- package/worker/subagents.py +6 -2
- package/worker/tool_catalog.py +5 -2
- package/worker/verification.py +113 -8
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""LanguageModel adapter for any OpenAI-compatible chat completions endpoint."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import copy
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import time
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import httpx
|
|
14
|
+
|
|
15
|
+
from mlai_review.application import usage as token_usage
|
|
16
|
+
from mlai_review.application.errors import PermanentReviewError, ReviewError
|
|
17
|
+
from mlai_review.application.ports import (
|
|
18
|
+
LanguageModel,
|
|
19
|
+
Message,
|
|
20
|
+
ToolCall,
|
|
21
|
+
ToolSpec,
|
|
22
|
+
Turn,
|
|
23
|
+
)
|
|
24
|
+
from mlai_review.infrastructure import messages
|
|
25
|
+
|
|
26
|
+
_log = logging.getLogger("mlai.orchestrator.model")
|
|
27
|
+
|
|
28
|
+
# Waits before retrying a rate-limited or failed call. Several readings and their refuter calls
|
|
29
|
+
# run at once, so a provider's per-minute limit is hit in bursts that clear within seconds;
|
|
30
|
+
# longer outages are left to the workflow's own retries.
|
|
31
|
+
RETRY_DELAYS = (3.0, 10.0)
|
|
32
|
+
# A large change reviewed in parts starts a dozen readings at once, then a refuter call per
|
|
33
|
+
# claim; providers rate-limit bursts (z.ai answered 429 to three parallel readings). Calls beyond
|
|
34
|
+
# this many wait their turn instead.
|
|
35
|
+
MAX_CONCURRENT_CALLS = 6
|
|
36
|
+
MAX_RETRY_AFTER_SECONDS = 30.0
|
|
37
|
+
# Asking again when an answer that must be JSON is not: GLM-5.3-Flash now and then ends a long
|
|
38
|
+
# answer in prose (in production two of sixty readings were lost this way, each a third of its
|
|
39
|
+
# review's readings). A fresh sample almost always parses.
|
|
40
|
+
INVALID_JSON_RETRIES = 1
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class OpenAICompatibleModel(LanguageModel):
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
*,
|
|
47
|
+
http: httpx.AsyncClient,
|
|
48
|
+
api_key: str,
|
|
49
|
+
model: str,
|
|
50
|
+
reasoning_effort: str | None = None,
|
|
51
|
+
max_tokens: int = 4_000,
|
|
52
|
+
extra_body: dict[str, Any] | None = None,
|
|
53
|
+
json_mode: bool = True,
|
|
54
|
+
retry_delays: tuple[float, ...] = RETRY_DELAYS,
|
|
55
|
+
max_concurrent: int = MAX_CONCURRENT_CALLS,
|
|
56
|
+
invalid_json_retries: int = INVALID_JSON_RETRIES,
|
|
57
|
+
) -> None:
|
|
58
|
+
self._http = http
|
|
59
|
+
self._api_key = api_key
|
|
60
|
+
self._model = model
|
|
61
|
+
self._reasoning_effort = reasoning_effort
|
|
62
|
+
self._max_tokens = max_tokens
|
|
63
|
+
# Provider-specific request fields, e.g. GLM's {"thinking": {"type": "disabled"}}.
|
|
64
|
+
self._extra_body = dict(extra_body or {})
|
|
65
|
+
# Some providers (Anthropic's OpenAI-compatible endpoint) reject response_format
|
|
66
|
+
# json_object; the prompt asks for JSON anyway and the parser tolerates fences.
|
|
67
|
+
self._json_mode = json_mode
|
|
68
|
+
self._retry_delays = retry_delays
|
|
69
|
+
self._invalid_json_retries = invalid_json_retries
|
|
70
|
+
# Calls in flight at once, across every reading, part and refuter call of a worker.
|
|
71
|
+
self._slots = asyncio.Semaphore(max_concurrent)
|
|
72
|
+
|
|
73
|
+
def thinking(self, reasoning_effort: str | None, max_tokens: int) -> OpenAICompatibleModel:
|
|
74
|
+
"""The same model asked to think at another effort, sharing this one's connection and
|
|
75
|
+
its limit on calls in flight, so that both together still respect the provider's."""
|
|
76
|
+
other = copy.copy(self)
|
|
77
|
+
other._reasoning_effort = reasoning_effort
|
|
78
|
+
other._max_tokens = max_tokens
|
|
79
|
+
return other
|
|
80
|
+
|
|
81
|
+
async def next_turn(self, messages: tuple[Message, ...], tools: tuple[ToolSpec, ...]) -> Turn:
|
|
82
|
+
body: dict[str, Any] = {
|
|
83
|
+
"model": self._model,
|
|
84
|
+
"max_tokens": self._max_tokens,
|
|
85
|
+
"messages": [_wire(m) for m in messages],
|
|
86
|
+
}
|
|
87
|
+
if tools:
|
|
88
|
+
body["tools"] = [
|
|
89
|
+
{
|
|
90
|
+
"type": "function",
|
|
91
|
+
"function": {
|
|
92
|
+
"name": t.name,
|
|
93
|
+
"description": t.description,
|
|
94
|
+
"parameters": t.parameters,
|
|
95
|
+
},
|
|
96
|
+
}
|
|
97
|
+
for t in tools
|
|
98
|
+
]
|
|
99
|
+
elif self._json_mode:
|
|
100
|
+
body["response_format"] = {"type": "json_object"}
|
|
101
|
+
if self._reasoning_effort:
|
|
102
|
+
body["reasoning_effort"] = self._reasoning_effort
|
|
103
|
+
body.update(self._extra_body)
|
|
104
|
+
choice = await self._complete(body)
|
|
105
|
+
message = choice["message"]
|
|
106
|
+
calls = tuple(
|
|
107
|
+
ToolCall(
|
|
108
|
+
id=str(c.get("id") or f"call-{i}"),
|
|
109
|
+
name=str(c["function"]["name"]),
|
|
110
|
+
arguments=_arguments(c["function"].get("arguments")),
|
|
111
|
+
)
|
|
112
|
+
for i, c in enumerate(message.get("tool_calls") or [])
|
|
113
|
+
)
|
|
114
|
+
if calls:
|
|
115
|
+
return Turn(tool_calls=calls)
|
|
116
|
+
content = message.get("content") or ""
|
|
117
|
+
if tools:
|
|
118
|
+
try:
|
|
119
|
+
return Turn(answer=_parse_json(content))
|
|
120
|
+
except ReviewError:
|
|
121
|
+
return Turn(text=content) # prose while tools were on offer: the caller re-asks
|
|
122
|
+
return Turn(answer=await self._json_answer(body, content))
|
|
123
|
+
|
|
124
|
+
async def _json_answer(self, body: dict[str, Any], content: str) -> dict[str, Any]:
|
|
125
|
+
"""The answer as a JSON object, asking again when it is not one."""
|
|
126
|
+
for _ in range(self._invalid_json_retries):
|
|
127
|
+
try:
|
|
128
|
+
return _parse_json(content)
|
|
129
|
+
except ReviewError as exc:
|
|
130
|
+
_log.warning(messages.MODEL_ASKING_AGAIN, exc)
|
|
131
|
+
content = (await self._complete(body))["message"].get("content") or ""
|
|
132
|
+
return _parse_json(content)
|
|
133
|
+
|
|
134
|
+
async def _complete(self, body: dict[str, Any]) -> dict[str, Any]:
|
|
135
|
+
for delay in self._retry_delays:
|
|
136
|
+
outcome = await self._attempt(body)
|
|
137
|
+
if not isinstance(outcome, _Retryable):
|
|
138
|
+
return outcome
|
|
139
|
+
wait = outcome.wait or delay
|
|
140
|
+
_log.warning(messages.MODEL_RETRYING, outcome.error, wait)
|
|
141
|
+
await asyncio.sleep(wait)
|
|
142
|
+
outcome = await self._attempt(body)
|
|
143
|
+
if isinstance(outcome, _Retryable):
|
|
144
|
+
raise outcome.error from outcome.error.__cause__
|
|
145
|
+
return outcome
|
|
146
|
+
|
|
147
|
+
async def _attempt(self, body: dict[str, Any]) -> dict[str, Any] | _Retryable:
|
|
148
|
+
token_usage.check_budget()
|
|
149
|
+
try:
|
|
150
|
+
async with self._slots:
|
|
151
|
+
token_usage.check_budget() # another reading may have used the rest while this waited
|
|
152
|
+
started = time.perf_counter()
|
|
153
|
+
response = await self._http.post(
|
|
154
|
+
"chat/completions",
|
|
155
|
+
headers={"Authorization": f"Bearer {self._api_key}"},
|
|
156
|
+
json=body,
|
|
157
|
+
)
|
|
158
|
+
except httpx.TransportError as exc:
|
|
159
|
+
error = ReviewError(messages.MODEL_UNREACHABLE.format(error=type(exc).__name__))
|
|
160
|
+
error.__cause__ = exc
|
|
161
|
+
return _Retryable(error)
|
|
162
|
+
if response.status_code in (408, 429) or response.status_code >= 500:
|
|
163
|
+
status = messages.MODEL_RETRYABLE_STATUS.format(status=response.status_code)
|
|
164
|
+
return _Retryable(ReviewError(status), _retry_after(response))
|
|
165
|
+
return self._accept(response, started)
|
|
166
|
+
|
|
167
|
+
def _accept(self, response: httpx.Response, started: float) -> dict[str, Any]:
|
|
168
|
+
if not response.is_success:
|
|
169
|
+
raise PermanentReviewError(messages.MODEL_REJECTED.format(status=response.status_code))
|
|
170
|
+
payload = response.json()
|
|
171
|
+
choice: dict[str, Any] = payload["choices"][0]
|
|
172
|
+
usage = payload.get("usage") or {}
|
|
173
|
+
# Billed even when the answer turns out unusable (cut off, not JSON): count it first.
|
|
174
|
+
token_usage.record(
|
|
175
|
+
_count(usage.get("prompt_tokens")),
|
|
176
|
+
_count((usage.get("prompt_tokens_details") or {}).get("cached_tokens")),
|
|
177
|
+
_count(usage.get("completion_tokens")),
|
|
178
|
+
usage if isinstance(usage, dict) else None,
|
|
179
|
+
)
|
|
180
|
+
_log.info(
|
|
181
|
+
messages.MODEL_CALL_LOG,
|
|
182
|
+
payload.get("model", self._model),
|
|
183
|
+
(time.perf_counter() - started) * 1000,
|
|
184
|
+
usage.get("prompt_tokens"),
|
|
185
|
+
(usage.get("prompt_tokens_details") or {}).get("cached_tokens"),
|
|
186
|
+
usage.get("completion_tokens"),
|
|
187
|
+
choice.get("finish_reason"),
|
|
188
|
+
)
|
|
189
|
+
if choice.get("finish_reason") == "length":
|
|
190
|
+
# Retrying with the same budget fails the same way: report it as permanent.
|
|
191
|
+
details = usage.get("completion_tokens_details") or {}
|
|
192
|
+
raise PermanentReviewError(
|
|
193
|
+
messages.MODEL_OUT_OF_TOKENS.format(
|
|
194
|
+
max_tokens=self._max_tokens, reasoning=details.get("reasoning_tokens", "?")
|
|
195
|
+
)
|
|
196
|
+
)
|
|
197
|
+
return choice
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
@dataclass(frozen=True, slots=True)
|
|
201
|
+
class _Retryable:
|
|
202
|
+
"""A call that failed in a way another try may not."""
|
|
203
|
+
|
|
204
|
+
error: ReviewError
|
|
205
|
+
wait: float | None = None
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _retry_after(response: httpx.Response) -> float | None:
|
|
209
|
+
"""The provider's own wait, when it gives one in seconds and it is short enough."""
|
|
210
|
+
try:
|
|
211
|
+
seconds = float(response.headers.get("retry-after", ""))
|
|
212
|
+
except ValueError:
|
|
213
|
+
return None
|
|
214
|
+
return seconds if 0 <= seconds <= MAX_RETRY_AFTER_SECONDS else None
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _wire(message: Message) -> dict[str, Any]:
|
|
218
|
+
match message.role:
|
|
219
|
+
case "assistant" if message.tool_calls:
|
|
220
|
+
return {
|
|
221
|
+
"role": "assistant",
|
|
222
|
+
"content": message.content or None,
|
|
223
|
+
"tool_calls": [
|
|
224
|
+
{
|
|
225
|
+
"id": c.id,
|
|
226
|
+
"type": "function",
|
|
227
|
+
"function": {"name": c.name, "arguments": json.dumps(c.arguments)},
|
|
228
|
+
}
|
|
229
|
+
for c in message.tool_calls
|
|
230
|
+
],
|
|
231
|
+
}
|
|
232
|
+
case "tool":
|
|
233
|
+
return {
|
|
234
|
+
"role": "tool",
|
|
235
|
+
"tool_call_id": message.tool_call_id,
|
|
236
|
+
"content": message.content,
|
|
237
|
+
}
|
|
238
|
+
return {"role": message.role, "content": message.content}
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _count(value: Any) -> int:
|
|
242
|
+
"""A token count from the provider's usage block; absent or malformed counts as 0."""
|
|
243
|
+
return value if isinstance(value, int) and value >= 0 else 0
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _arguments(raw: Any) -> dict[str, Any]:
|
|
247
|
+
"""Tool arguments arrive as a JSON string; a malformed one becomes an empty call, which
|
|
248
|
+
the tool answers with an error the model can read."""
|
|
249
|
+
if isinstance(raw, dict):
|
|
250
|
+
return raw
|
|
251
|
+
try:
|
|
252
|
+
value = json.loads(raw or "{}")
|
|
253
|
+
except json.JSONDecodeError:
|
|
254
|
+
return {}
|
|
255
|
+
return value if isinstance(value, dict) else {}
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _parse_json(text: str) -> dict[str, Any]:
|
|
259
|
+
"""Models in JSON mode usually return bare JSON; tolerate a fenced block as well."""
|
|
260
|
+
cleaned = text.strip()
|
|
261
|
+
if cleaned.startswith("```"):
|
|
262
|
+
cleaned = cleaned.strip("`").removeprefix("json").strip()
|
|
263
|
+
try:
|
|
264
|
+
value = json.loads(cleaned)
|
|
265
|
+
except json.JSONDecodeError as exc:
|
|
266
|
+
raise ReviewError(messages.MODEL_NOT_JSON) from exc
|
|
267
|
+
if not isinstance(value, dict):
|
|
268
|
+
raise ReviewError(messages.MODEL_NOT_OBJECT)
|
|
269
|
+
return value
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Source-structure adapters: how a file's outline is parsed."""
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Outlines files without a parser, from indentation (Python) or braces (C-like languages).
|
|
2
|
+
Used when tree-sitter is not installed where the review runs, or a grammar cannot be loaded:
|
|
3
|
+
less exact than a syntax tree, but enough to show the whole function around a change and to
|
|
4
|
+
find what it calls and what calls it."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
from pathlib import PurePosixPath
|
|
10
|
+
|
|
11
|
+
from mlai_review.application.ports import Outliner
|
|
12
|
+
from mlai_review.domain.structure import Definition, DefinitionKind, Outline
|
|
13
|
+
|
|
14
|
+
_PY_DEF = re.compile(r"^(?P<indent>[ \t]*)(?:async[ \t]+)?def[ \t]+(?P<name>\w+)")
|
|
15
|
+
_PY_CLASS = re.compile(r"^(?P<indent>[ \t]*)class[ \t]+(?P<name>\w+)")
|
|
16
|
+
_PY_IMPORT = re.compile(r"^[ \t]*(?:import[ \t]+\w|from[ \t]+[\w.]+[ \t]+import\b)")
|
|
17
|
+
|
|
18
|
+
_C_LIKE = frozenset(
|
|
19
|
+
{".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".go", ".java", ".kt", ".kts", ".rs", ".cs",
|
|
20
|
+
".php", ".c", ".h", ".cc", ".cpp", ".hpp", ".swift", ".scala", ".dart"}
|
|
21
|
+
)
|
|
22
|
+
_KEYWORDS = frozenset(
|
|
23
|
+
{"if", "for", "while", "switch", "catch", "return", "else", "do", "try", "new", "typeof",
|
|
24
|
+
"await", "function", "with", "synchronized", "using", "lock", "foreach", "match", "elif"}
|
|
25
|
+
)
|
|
26
|
+
_TYPE = re.compile(
|
|
27
|
+
r"^\s*(?:export\s+)?(?:default\s+)?(?:public\s+|private\s+|protected\s+|internal\s+|"
|
|
28
|
+
r"abstract\s+|final\s+|sealed\s+|static\s+|pub(?:\([\w ]+\))?\s+|data\s+|open\s+)*"
|
|
29
|
+
r"(?:class|interface|struct|enum|trait|impl|object|record|namespace|module)\s+"
|
|
30
|
+
r"(?:<[^>]*>\s*)?(?P<name>[A-Za-z_$][\w$]*)"
|
|
31
|
+
)
|
|
32
|
+
_GO_TYPE = re.compile(r"^\s*type\s+(?P<name>\w+)\s+(?:struct|interface)\b")
|
|
33
|
+
_FUNCTION = re.compile(
|
|
34
|
+
r"^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?(?:pub(?:\([\w ]+\))?\s+)?"
|
|
35
|
+
r"(?:function\*?|func|fn|fun|def)\s+(?:\([^)]*\)\s*)?(?P<name>[A-Za-z_$][\w$]*)"
|
|
36
|
+
)
|
|
37
|
+
_BOUND = re.compile(
|
|
38
|
+
r"^\s*(?:export\s+)?(?:const|let|var)\s+(?P<name>[A-Za-z_$][\w$]*)\s*(?::[^=]+)?=\s*"
|
|
39
|
+
r"(?:async\s+)?(?:function\b|\([^)]*\)\s*(?::[^=]+)?=>|[A-Za-z_$][\w$]*\s*=>)"
|
|
40
|
+
)
|
|
41
|
+
# A method or a C-style function: a name, its parameters, then a body on this line or the next.
|
|
42
|
+
_METHOD = re.compile(
|
|
43
|
+
r"^\s*(?:(?:public|private|protected|static|async|override|virtual|final|abstract|"
|
|
44
|
+
r"internal|readonly|get|set|inline|const|unsafe|extern)\s+)*"
|
|
45
|
+
r"(?:[\w$<>\[\],.*&:? ]+\s+)?\*?(?P<name>[A-Za-z_$][\w$]*)\s*\([^;]*\)\s*(?:[:\-]>?[^{;=]*)?\{?\s*$"
|
|
46
|
+
)
|
|
47
|
+
_C_IMPORT = re.compile(
|
|
48
|
+
r"^\s*(?:import\b|from\s+['\"]|#include\b|use\s+[\w:{]|using\s+[\w.]+;|"
|
|
49
|
+
r"(?:const|let|var)\s+[\w{}, ]+=\s*require\()"
|
|
50
|
+
)
|
|
51
|
+
_MAX_NAME_CHARS = 60
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class HeuristicOutliner(Outliner):
|
|
55
|
+
def outline(self, path: str, text: str) -> Outline | None:
|
|
56
|
+
suffix = PurePosixPath(path).suffix.lower()
|
|
57
|
+
if suffix in (".py", ".pyi"):
|
|
58
|
+
return _python(text.splitlines())
|
|
59
|
+
if suffix in _C_LIKE:
|
|
60
|
+
return _braces(text.splitlines())
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _python(lines: list[str]) -> Outline:
|
|
65
|
+
found: list[tuple[DefinitionKind, str, int, int]] = [] # kind, name, start, indent
|
|
66
|
+
imports: list[int] = []
|
|
67
|
+
for number, line in enumerate(lines, 1):
|
|
68
|
+
if _PY_IMPORT.match(line):
|
|
69
|
+
imports.append(number)
|
|
70
|
+
match = _PY_DEF.match(line) or _PY_CLASS.match(line)
|
|
71
|
+
if match:
|
|
72
|
+
kind = DefinitionKind.FUNCTION if "def" in line.split(match["name"])[0] else (
|
|
73
|
+
DefinitionKind.TYPE
|
|
74
|
+
)
|
|
75
|
+
found.append((kind, match["name"], number, _width(match["indent"])))
|
|
76
|
+
definitions = []
|
|
77
|
+
for kind, name, start, indent in found:
|
|
78
|
+
end = start
|
|
79
|
+
for number in range(start + 1, len(lines) + 1):
|
|
80
|
+
line = lines[number - 1]
|
|
81
|
+
if not line.strip():
|
|
82
|
+
continue
|
|
83
|
+
if _width(line[: len(line) - len(line.lstrip())]) <= indent and not _continues(line):
|
|
84
|
+
break
|
|
85
|
+
end = number
|
|
86
|
+
definitions.append(Definition(kind, name[:_MAX_NAME_CHARS], start, end, 0))
|
|
87
|
+
return Outline(definitions=_with_depths(definitions), imports=tuple(imports))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _continues(line: str) -> bool:
|
|
91
|
+
"""A closing bracket at the def's own indent still belongs to it (multi-line signatures)."""
|
|
92
|
+
return line.strip()[:1] in (")", "]", "}")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _width(indent: str) -> int:
|
|
96
|
+
return len(indent.replace("\t", " "))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _braces(lines: list[str]) -> Outline:
|
|
100
|
+
imports: list[int] = []
|
|
101
|
+
starts: list[tuple[DefinitionKind, str, int]] = []
|
|
102
|
+
for number, line in enumerate(lines, 1):
|
|
103
|
+
if _C_IMPORT.match(line):
|
|
104
|
+
imports.append(number)
|
|
105
|
+
continue
|
|
106
|
+
kind_name = _definition(line, lines[number] if number < len(lines) else "")
|
|
107
|
+
if kind_name is not None:
|
|
108
|
+
starts.append((kind_name[0], kind_name[1], number))
|
|
109
|
+
definitions = []
|
|
110
|
+
for kind, name, start in starts:
|
|
111
|
+
end = _block_end(lines, start)
|
|
112
|
+
if end is not None:
|
|
113
|
+
definitions.append(Definition(kind, name[:_MAX_NAME_CHARS], start, end, 0))
|
|
114
|
+
return Outline(definitions=_with_depths(definitions), imports=tuple(imports))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _definition(line: str, following: str) -> tuple[DefinitionKind, str] | None:
|
|
118
|
+
stripped = line.strip()
|
|
119
|
+
if not stripped or stripped.startswith(("//", "/*", "*", "#")):
|
|
120
|
+
return None
|
|
121
|
+
for pattern, kind in ((_TYPE, DefinitionKind.TYPE), (_GO_TYPE, DefinitionKind.TYPE)):
|
|
122
|
+
match = pattern.match(line)
|
|
123
|
+
if match:
|
|
124
|
+
return kind, match["name"]
|
|
125
|
+
for pattern in (_FUNCTION, _BOUND):
|
|
126
|
+
match = pattern.match(line)
|
|
127
|
+
if match:
|
|
128
|
+
return DefinitionKind.FUNCTION, match["name"]
|
|
129
|
+
match = _METHOD.match(line)
|
|
130
|
+
if match and match["name"] not in _KEYWORDS:
|
|
131
|
+
if stripped.endswith("{") or following.strip().startswith("{"):
|
|
132
|
+
return DefinitionKind.FUNCTION, match["name"]
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _block_end(lines: list[str], start: int) -> int | None:
|
|
137
|
+
"""The line closing the first brace opened at or after `start`, skipping strings and
|
|
138
|
+
comments roughly; None when no block opens within a few lines (a declaration only)."""
|
|
139
|
+
depth = 0
|
|
140
|
+
opened = False
|
|
141
|
+
in_block_comment = False
|
|
142
|
+
for number in range(start, len(lines) + 1):
|
|
143
|
+
text = lines[number - 1]
|
|
144
|
+
i = 0
|
|
145
|
+
quote = ""
|
|
146
|
+
while i < len(text):
|
|
147
|
+
ch = text[i]
|
|
148
|
+
pair = text[i : i + 2]
|
|
149
|
+
if in_block_comment:
|
|
150
|
+
if pair == "*/":
|
|
151
|
+
in_block_comment = False
|
|
152
|
+
i += 1
|
|
153
|
+
elif quote:
|
|
154
|
+
if ch == "\\":
|
|
155
|
+
i += 1
|
|
156
|
+
elif ch == quote:
|
|
157
|
+
quote = ""
|
|
158
|
+
elif pair == "//":
|
|
159
|
+
break
|
|
160
|
+
elif pair == "/*":
|
|
161
|
+
in_block_comment = True
|
|
162
|
+
i += 1
|
|
163
|
+
elif ch in "\"'`":
|
|
164
|
+
quote = ch
|
|
165
|
+
elif ch == "{":
|
|
166
|
+
depth += 1
|
|
167
|
+
opened = True
|
|
168
|
+
elif ch == "}":
|
|
169
|
+
depth -= 1
|
|
170
|
+
if opened and depth == 0:
|
|
171
|
+
return number
|
|
172
|
+
elif ch == ";" and not opened and depth == 0:
|
|
173
|
+
return None # `function f();`, an abstract method, a type alias
|
|
174
|
+
i += 1
|
|
175
|
+
if not opened and number - start >= 3:
|
|
176
|
+
return None
|
|
177
|
+
return None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _with_depths(definitions: list[Definition]) -> tuple[Definition, ...]:
|
|
181
|
+
ordered = sorted(definitions, key=lambda d: (d.start, -d.end))
|
|
182
|
+
out: list[Definition] = []
|
|
183
|
+
for d in ordered:
|
|
184
|
+
depth = sum(1 for p in out if p.start <= d.start and d.end <= p.end and p is not d)
|
|
185
|
+
out.append(Definition(d.kind, d.name, d.start, d.end, depth))
|
|
186
|
+
return tuple(out)
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Outlines files with tree-sitter grammars. Unknown languages get no outline, and the context
|
|
2
|
+
falls back to windows around each change. So does a language whose grammar cannot be loaded:
|
|
3
|
+
an outline is a refinement, never a reason to fail a review."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
from pathlib import PurePosixPath
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
try: # Optional where the review runs: without it, the heuristic outliner stands in.
|
|
12
|
+
from tree_sitter_language_pack import PackConfig, configure, get_parser, prefetch
|
|
13
|
+
except ImportError: # pragma: no cover - depends on the machine
|
|
14
|
+
PackConfig = configure = get_parser = prefetch = None # type: ignore[assignment,misc]
|
|
15
|
+
|
|
16
|
+
from mlai_review.application.ports import Outliner
|
|
17
|
+
from mlai_review.infrastructure.structure.heuristic import HeuristicOutliner
|
|
18
|
+
from mlai_review.domain.structure import Definition, DefinitionKind, Outline
|
|
19
|
+
from mlai_review.infrastructure import messages
|
|
20
|
+
|
|
21
|
+
_log = logging.getLogger("mlai.orchestrator.outline")
|
|
22
|
+
|
|
23
|
+
_LANGUAGES = {
|
|
24
|
+
".py": "python",
|
|
25
|
+
".pyi": "python",
|
|
26
|
+
".ts": "typescript",
|
|
27
|
+
".tsx": "tsx",
|
|
28
|
+
".js": "javascript",
|
|
29
|
+
".jsx": "javascript",
|
|
30
|
+
".mjs": "javascript",
|
|
31
|
+
".cjs": "javascript",
|
|
32
|
+
".go": "go",
|
|
33
|
+
".java": "java",
|
|
34
|
+
".kt": "kotlin",
|
|
35
|
+
".kts": "kotlin",
|
|
36
|
+
".rs": "rust",
|
|
37
|
+
".rb": "ruby",
|
|
38
|
+
".cs": "csharp",
|
|
39
|
+
".php": "php",
|
|
40
|
+
".c": "c",
|
|
41
|
+
".h": "c",
|
|
42
|
+
".cc": "cpp",
|
|
43
|
+
".cpp": "cpp",
|
|
44
|
+
".hpp": "cpp",
|
|
45
|
+
".lua": "lua",
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
# Every grammar the outliner may ask for; the image prefetches exactly these at build time.
|
|
49
|
+
SUPPORTED_LANGUAGES: tuple[str, ...] = tuple(sorted(set(_LANGUAGES.values())))
|
|
50
|
+
|
|
51
|
+
_FUNCTIONS = frozenset(
|
|
52
|
+
{
|
|
53
|
+
"function_definition",
|
|
54
|
+
"function_declaration",
|
|
55
|
+
"generator_function_declaration",
|
|
56
|
+
"method_definition",
|
|
57
|
+
"method_declaration",
|
|
58
|
+
"constructor_declaration",
|
|
59
|
+
"function_item",
|
|
60
|
+
"method",
|
|
61
|
+
"singleton_method",
|
|
62
|
+
"local_function",
|
|
63
|
+
}
|
|
64
|
+
)
|
|
65
|
+
_TYPES = frozenset(
|
|
66
|
+
{
|
|
67
|
+
"class_definition",
|
|
68
|
+
"class_declaration",
|
|
69
|
+
"class_specifier",
|
|
70
|
+
"struct_specifier",
|
|
71
|
+
"struct_item",
|
|
72
|
+
"enum_item",
|
|
73
|
+
"enum_declaration",
|
|
74
|
+
"trait_item",
|
|
75
|
+
"impl_item",
|
|
76
|
+
"interface_declaration",
|
|
77
|
+
"record_declaration",
|
|
78
|
+
"object_declaration",
|
|
79
|
+
"class",
|
|
80
|
+
"module",
|
|
81
|
+
"type_declaration",
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
# `const handler = async () => {...}` and friends: a function bound to a name.
|
|
85
|
+
_FUNCTION_VALUES = frozenset({"arrow_function", "function_expression", "function"})
|
|
86
|
+
_IMPORTS = frozenset(
|
|
87
|
+
{
|
|
88
|
+
"import_statement",
|
|
89
|
+
"import_from_statement",
|
|
90
|
+
"import_declaration",
|
|
91
|
+
"import_header",
|
|
92
|
+
"use_declaration",
|
|
93
|
+
"using_directive",
|
|
94
|
+
"preproc_include",
|
|
95
|
+
"namespace_use_declaration",
|
|
96
|
+
}
|
|
97
|
+
)
|
|
98
|
+
_MAX_NAME_CHARS = 60
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class TreeSitterOutliner(Outliner):
|
|
102
|
+
"""Grammars come from the language pack's cache. In the image that cache is filled at build
|
|
103
|
+
time (see `prefetch_all`), so a review never downloads anything."""
|
|
104
|
+
|
|
105
|
+
def __init__(self, cache_dir: str | None = None) -> None:
|
|
106
|
+
if cache_dir and configure is not None:
|
|
107
|
+
configure(PackConfig(cache_dir=cache_dir))
|
|
108
|
+
self._fallback = HeuristicOutliner()
|
|
109
|
+
self._missing: set[str] = set()
|
|
110
|
+
|
|
111
|
+
def outline(self, path: str, text: str) -> Outline | None:
|
|
112
|
+
language = _LANGUAGES.get(PurePosixPath(path).suffix.lower())
|
|
113
|
+
if language is None:
|
|
114
|
+
return None
|
|
115
|
+
if get_parser is None or language in self._missing:
|
|
116
|
+
return self._fallback.outline(path, text)
|
|
117
|
+
try:
|
|
118
|
+
parser = get_parser(language)
|
|
119
|
+
except Exception as exc:
|
|
120
|
+
_log.warning(messages.GRAMMAR_UNAVAILABLE, language, exc)
|
|
121
|
+
self._missing.add(language)
|
|
122
|
+
return self._fallback.outline(path, text)
|
|
123
|
+
tree = parser.parse(text.encode())
|
|
124
|
+
definitions: list[Definition] = []
|
|
125
|
+
imports: list[int] = []
|
|
126
|
+
# Explicit stack: deep expressions would overflow a recursive walk.
|
|
127
|
+
stack: list[tuple[Any, int]] = [(child, 0) for child in reversed(tree.root_node.children)]
|
|
128
|
+
while stack:
|
|
129
|
+
node, depth = stack.pop()
|
|
130
|
+
kind = _kind(node) if node.is_named else None # keywords like `class` are not named
|
|
131
|
+
if kind is not None:
|
|
132
|
+
definitions.append(
|
|
133
|
+
Definition(
|
|
134
|
+
kind=kind,
|
|
135
|
+
name=_name(node),
|
|
136
|
+
start=node.start_point.row + 1,
|
|
137
|
+
end=node.end_point.row + 1,
|
|
138
|
+
depth=depth,
|
|
139
|
+
)
|
|
140
|
+
)
|
|
141
|
+
depth += 1
|
|
142
|
+
elif node.type in _IMPORTS:
|
|
143
|
+
imports.append(node.start_point.row + 1)
|
|
144
|
+
stack.extend((child, depth) for child in reversed(node.children))
|
|
145
|
+
definitions.sort(key=lambda d: (d.start, d.depth))
|
|
146
|
+
return Outline(definitions=tuple(definitions), imports=tuple(sorted(imports)))
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _kind(node: Any) -> DefinitionKind | None:
|
|
150
|
+
if node.type in _FUNCTIONS:
|
|
151
|
+
return DefinitionKind.FUNCTION
|
|
152
|
+
if node.type in _TYPES:
|
|
153
|
+
return DefinitionKind.TYPE
|
|
154
|
+
if node.type == "variable_declarator":
|
|
155
|
+
value = node.child_by_field_name("value")
|
|
156
|
+
if value is not None and value.type in _FUNCTION_VALUES:
|
|
157
|
+
return DefinitionKind.FUNCTION
|
|
158
|
+
return None
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _name(node: Any) -> str:
|
|
162
|
+
named = node.child_by_field_name("name")
|
|
163
|
+
if named is None and node.type == "function_definition": # C and C++
|
|
164
|
+
declarator = node.child_by_field_name("declarator")
|
|
165
|
+
while declarator is not None and declarator.type != "function_declarator":
|
|
166
|
+
declarator = declarator.child_by_field_name("declarator")
|
|
167
|
+
if declarator is not None:
|
|
168
|
+
named = declarator.child_by_field_name("declarator")
|
|
169
|
+
source = named if named is not None else node
|
|
170
|
+
return str(source.text.decode("utf-8", "replace").splitlines()[0][:_MAX_NAME_CHARS])
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def available() -> bool:
|
|
174
|
+
return get_parser is not None
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def prefetch_all(cache_dir: str) -> None:
|
|
178
|
+
"""Download every supported grammar into `cache_dir`. Run while building the image."""
|
|
179
|
+
configure(PackConfig(cache_dir=cache_dir))
|
|
180
|
+
prefetch(list(SUPPORTED_LANGUAGES))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
if __name__ == "__main__": # python -m ...tree_sitter <cache_dir>
|
|
184
|
+
import sys
|
|
185
|
+
|
|
186
|
+
prefetch_all(sys.argv[1])
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Workspace adapters: where the reviewed commit's files live while a review runs."""
|