mlai-code 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/checkpoints.mjs +217 -0
- package/src/cli.mjs +178 -31
- package/src/cloud.mjs +2 -0
- package/src/engine.mjs +1 -1
- package/src/headless.mjs +25 -9
- package/src/home.mjs +3 -2
- package/src/host.mjs +141 -13
- package/src/ide-quick.mjs +147 -0
- package/src/ide.mjs +182 -0
- package/src/netbox.mjs +1 -1
- package/src/parity.mjs +20 -1
- package/src/relay.mjs +1 -1
- package/src/render.mjs +12 -3
- package/src/runtime.mjs +23 -1
- package/src/sandbox.mjs +19 -3
- package/src/server.mjs +46 -7
- package/src/session.mjs +53 -10
- package/src/shared-data.mjs +1 -1
- package/src/tui/acp.mjs +121 -19
- package/src/tui/app.mjs +24 -5
- package/src/tui/chat.mjs +43 -16
- package/src/tui/cmd-config.mjs +3 -1
- package/src/tui/cmd-core.mjs +11 -7
- package/src/tui/cmd-extensibility.mjs +13 -2
- package/src/tui/commands.mjs +2 -1
- package/src/tui/extensions-local.mjs +2 -0
- package/src/tui/mcp-config.mjs +22 -4
- package/src/tui/review-cli.mjs +15 -4
- package/src/tui/snapshots.mjs +57 -16
- package/src/tui/statusline.mjs +2 -1
- package/src/two-words.mjs +14 -8
- package/src/view-sync.mjs +48 -0
- package/src/wakeups.mjs +2 -2
- package/src/winenv.mjs +1 -1
- package/src/workspace-api.mjs +21 -4
- package/vendor/client-sdk/index.cjs +5 -4
- package/vendor/cloud-cli/mlai.mjs +184 -28
- package/worker/architecture_worker.py +33 -4
- package/worker/browser_tool.py +19 -1
- package/worker/build_planner.py +75 -7
- package/worker/chat_fork.py +4 -1
- package/worker/checkpoints.py +71 -8
- package/worker/cloud_pty.py +91 -2
- package/worker/command_safety.py +80 -3
- package/worker/compaction_brief.py +186 -0
- package/worker/connected_turn.py +30 -0
- package/worker/context_manager.py +6 -0
- package/worker/conversation.py +139 -12
- package/worker/deck_render.js +11 -0
- package/worker/deck_tools.py +825 -0
- package/worker/git_sync.py +42 -17
- package/worker/goals.py +4 -0
- package/worker/image_generation.py +102 -23
- package/worker/kimi_bridge.py +580 -52
- package/worker/kimi_gates.py +74 -6
- package/worker/kimi_hook.py +213 -19
- package/worker/media_tools.py +182 -0
- package/worker/memory.py +2 -2
- package/worker/mlai_hooks.py +314 -5
- package/worker/mlai_mcp.py +49 -11
- package/worker/mlai_review/README.md +35 -0
- package/worker/mlai_review/__init__.py +1 -0
- package/worker/mlai_review/__main__.py +261 -0
- package/worker/mlai_review/_compat.py +29 -0
- package/worker/mlai_review/analysis/__init__.py +3 -0
- package/worker/mlai_review/analysis/detect.py +50 -0
- package/worker/mlai_review/analysis/main.py +128 -0
- package/worker/mlai_review/analysis/messages.py +8 -0
- package/worker/mlai_review/analysis/tools.py +194 -0
- package/worker/mlai_review/application/__init__.py +1 -0
- package/worker/mlai_review/application/asking.py +53 -0
- package/worker/mlai_review/application/challengers.py +142 -0
- package/worker/mlai_review/application/context_builder.py +194 -0
- package/worker/mlai_review/application/errors.py +17 -0
- package/worker/mlai_review/application/feedback.py +52 -0
- package/worker/mlai_review/application/matching.py +46 -0
- package/worker/mlai_review/application/messages.py +41 -0
- package/worker/mlai_review/application/ports.py +222 -0
- package/worker/mlai_review/application/prompts.py +342 -0
- package/worker/mlai_review/application/review_service.py +200 -0
- package/worker/mlai_review/application/reviewers.py +438 -0
- package/worker/mlai_review/application/streaming.py +91 -0
- package/worker/mlai_review/application/tools.py +138 -0
- package/worker/mlai_review/application/usage.py +145 -0
- package/worker/mlai_review/application/write_check.py +46 -0
- package/worker/mlai_review/contracts/__init__.py +5 -0
- package/worker/mlai_review/contracts/analysis.py +57 -0
- package/worker/mlai_review/contracts/review.py +42 -0
- package/worker/mlai_review/domain/__init__.py +4 -0
- package/worker/mlai_review/domain/changes.py +77 -0
- package/worker/mlai_review/domain/context.py +147 -0
- package/worker/mlai_review/domain/diagnostics.py +239 -0
- package/worker/mlai_review/domain/feedback.py +216 -0
- package/worker/mlai_review/domain/findings.py +505 -0
- package/worker/mlai_review/domain/lifecycle.py +85 -0
- package/worker/mlai_review/domain/partition.py +42 -0
- package/worker/mlai_review/domain/relations.py +166 -0
- package/worker/mlai_review/domain/relevance.py +202 -0
- package/worker/mlai_review/domain/report.py +99 -0
- package/worker/mlai_review/domain/structure.py +56 -0
- package/worker/mlai_review/domain/writes.py +86 -0
- package/worker/mlai_review/eval.py +221 -0
- package/worker/mlai_review/eval_cases/clean-rename.json +11 -0
- package/worker/mlai_review/eval_cases/missing-ownership-check.json +22 -0
- package/worker/mlai_review/eval_cases/off-by-one.json +22 -0
- package/worker/mlai_review/eval_cases/sql-injection.json +20 -0
- package/worker/mlai_review/feedback_report.py +60 -0
- package/worker/mlai_review/infrastructure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/none.py +12 -0
- package/worker/mlai_review/infrastructure/analysis/translate.py +17 -0
- package/worker/mlai_review/infrastructure/github/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/github/client.py +445 -0
- package/worker/mlai_review/infrastructure/github/feedback.py +212 -0
- package/worker/mlai_review/infrastructure/github/http.py +31 -0
- package/worker/mlai_review/infrastructure/github/rendering.py +398 -0
- package/worker/mlai_review/infrastructure/github/token.py +20 -0
- package/worker/mlai_review/infrastructure/local/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/local/analyzer.py +48 -0
- package/worker/mlai_review/infrastructure/local/events.py +100 -0
- package/worker/mlai_review/infrastructure/local/git_reader.py +233 -0
- package/worker/mlai_review/infrastructure/local/publisher.py +138 -0
- package/worker/mlai_review/infrastructure/messages.py +54 -0
- package/worker/mlai_review/infrastructure/models/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/models/openai_compatible.py +269 -0
- package/worker/mlai_review/infrastructure/structure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/structure/heuristic.py +186 -0
- package/worker/mlai_review/infrastructure/structure/tree_sitter.py +186 -0
- package/worker/mlai_review/infrastructure/workspace/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/workspace/local.py +105 -0
- package/worker/mlai_review/infrastructure/workspace/memory.py +77 -0
- package/worker/mlai_review/infrastructure/workspace/paths.py +17 -0
- package/worker/mlai_review/pipeline.py +89 -0
- package/worker/mlai_review/runtime.py +194 -0
- package/worker/mlai_review/settings.py +139 -0
- package/worker/model_stream.py +21 -7
- package/worker/offline_guard.py +1 -1
- package/worker/preview.py +62 -4
- package/worker/product_facts.py +20 -0
- package/worker/project_context.py +33 -7
- package/worker/reply_language.py +92 -10
- package/worker/repo_policy.py +5 -0
- package/worker/roles.json +199 -37
- package/worker/roles.py +271 -9
- package/worker/runner.py +366 -53
- package/worker/runtime_control.py +64 -3
- package/worker/secret_scan.py +3 -2
- package/worker/server_memory.py +122 -0
- package/worker/site_check.py +279 -0
- package/worker/skill_library.py +15 -3
- package/worker/skill_pack.py +170 -16
- package/worker/subagents.py +6 -2
- package/worker/tool_catalog.py +4 -3
- package/worker/verification.py +156 -19
- package/worker/wakeups.py +56 -4
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""The reviewer's eval harness: planted defects in small repositories, reviewed end to end
|
|
2
|
+
(context, readings, grouping, refuter, policy), scored by recall (planted issues found) and
|
|
3
|
+
precision (published findings that are real). Cases live in eval_cases/*.json:
|
|
4
|
+
|
|
5
|
+
{"name", "mode", "before": {path: text}, "after": {path: text},
|
|
6
|
+
"expected": [{"path", "line", "keywords": [...]}]}
|
|
7
|
+
|
|
8
|
+
`--record` calls the model named in `--request` and saves every reply next to the cases, so the
|
|
9
|
+
same run can be replayed offline, without a model and with the same result (the default)."""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import asyncio
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import subprocess
|
|
18
|
+
import tempfile
|
|
19
|
+
from dataclasses import asdict, dataclass
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from mlai_review.application.ports import LanguageModel, Message, ToolCall, ToolSpec, Turn
|
|
24
|
+
from mlai_review.application.review_service import ReviewService
|
|
25
|
+
from mlai_review.application.context_builder import ContextBuilder
|
|
26
|
+
from mlai_review.domain.findings import Finding
|
|
27
|
+
from mlai_review.domain.relevance import ReviewLimits
|
|
28
|
+
from mlai_review.domain.report import ReviewReport
|
|
29
|
+
from mlai_review.infrastructure.local.events import EventLog
|
|
30
|
+
from mlai_review.infrastructure.local.git_reader import LocalGitReader
|
|
31
|
+
from mlai_review.infrastructure.local.publisher import EventPublisher
|
|
32
|
+
from mlai_review.infrastructure.structure.tree_sitter import TreeSitterOutliner
|
|
33
|
+
from mlai_review.runtime import CheckoutProvider, reviewer_for
|
|
34
|
+
from mlai_review.settings import ModelSettings, for_mode
|
|
35
|
+
|
|
36
|
+
CASES = Path(__file__).parent / "eval_cases"
|
|
37
|
+
NEAR_LINES = 3
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True, slots=True)
|
|
41
|
+
class CaseResult:
|
|
42
|
+
name: str
|
|
43
|
+
expected: int
|
|
44
|
+
found: int
|
|
45
|
+
published: int
|
|
46
|
+
correct: int
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True, slots=True)
|
|
50
|
+
class Score:
|
|
51
|
+
cases: tuple[CaseResult, ...]
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def recall(self) -> float:
|
|
55
|
+
expected = sum(c.expected for c in self.cases)
|
|
56
|
+
return 1.0 if expected == 0 else sum(c.found for c in self.cases) / expected
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def precision(self) -> float:
|
|
60
|
+
published = sum(c.published for c in self.cases)
|
|
61
|
+
return 1.0 if published == 0 else sum(c.correct for c in self.cases) / published
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def key(messages: tuple[Message, ...], tools: tuple[ToolSpec, ...]) -> str:
|
|
65
|
+
wire = [[m.role, m.content, [asdict(c) for c in m.tool_calls], m.tool_call_id] for m in messages]
|
|
66
|
+
blob = json.dumps([wire, [t.name for t in tools]], sort_keys=True)
|
|
67
|
+
return hashlib.sha256(blob.encode()).hexdigest()[:24]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _turn_json(turn: Turn) -> dict[str, Any]:
|
|
71
|
+
return {"answer": turn.answer, "text": turn.text, "calls": [asdict(c) for c in turn.tool_calls]}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _turn(data: dict[str, Any]) -> Turn:
|
|
75
|
+
return Turn(
|
|
76
|
+
tool_calls=tuple(ToolCall(**c) for c in data.get("calls") or ()),
|
|
77
|
+
answer=data.get("answer"),
|
|
78
|
+
text=data.get("text") or "",
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class RecordingModel(LanguageModel):
|
|
83
|
+
def __init__(self, inner: LanguageModel, replies: dict[str, Any]) -> None:
|
|
84
|
+
self._inner = inner
|
|
85
|
+
self.replies = replies
|
|
86
|
+
|
|
87
|
+
async def next_turn(self, messages: tuple[Message, ...], tools: tuple[ToolSpec, ...]) -> Turn:
|
|
88
|
+
turn = await self._inner.next_turn(messages, tools)
|
|
89
|
+
self.replies[key(messages, tools)] = _turn_json(turn)
|
|
90
|
+
return turn
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class ReplayModel(LanguageModel):
|
|
94
|
+
"""Answers from recorded replies; a question never recorded gets an empty answer, counted
|
|
95
|
+
in `misses` (the prompts changed since the recording)."""
|
|
96
|
+
|
|
97
|
+
def __init__(self, replies: dict[str, Any]) -> None:
|
|
98
|
+
self._replies = replies
|
|
99
|
+
self.misses = 0
|
|
100
|
+
|
|
101
|
+
async def next_turn(self, messages: tuple[Message, ...], tools: tuple[ToolSpec, ...]) -> Turn:
|
|
102
|
+
data = self._replies.get(key(messages, tools))
|
|
103
|
+
if data is None:
|
|
104
|
+
self.misses += 1
|
|
105
|
+
return Turn(answer={"findings": []})
|
|
106
|
+
return _turn(data)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def matches(expected: dict[str, Any], finding: Finding) -> bool:
|
|
110
|
+
if finding.path != expected["path"]:
|
|
111
|
+
return False
|
|
112
|
+
if abs(finding.line - int(expected["line"])) <= NEAR_LINES:
|
|
113
|
+
return True
|
|
114
|
+
words = f"{finding.title} {finding.explanation}".lower()
|
|
115
|
+
return any(k.lower() in words for k in expected.get("keywords") or ())
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def score_case(case: dict[str, Any], report: ReviewReport) -> CaseResult:
|
|
119
|
+
published = report.findings
|
|
120
|
+
expected = case.get("expected") or []
|
|
121
|
+
found = sum(1 for e in expected if any(matches(e, f) for f in published))
|
|
122
|
+
correct = sum(1 for f in published if any(matches(e, f) for e in expected))
|
|
123
|
+
return CaseResult(case["name"], len(expected), found, len(published), correct)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def checkout(case: dict[str, Any], into: Path) -> None:
|
|
127
|
+
def git(*args: str) -> None:
|
|
128
|
+
subprocess.run(["git", "-C", str(into), *args], check=True, capture_output=True)
|
|
129
|
+
|
|
130
|
+
git("init", "-q")
|
|
131
|
+
for path, text in case["before"].items():
|
|
132
|
+
(into / path).parent.mkdir(parents=True, exist_ok=True)
|
|
133
|
+
(into / path).write_text(text)
|
|
134
|
+
git("add", "-A")
|
|
135
|
+
git("-c", "user.name=eval", "-c", "user.email=eval@localhost", "commit", "-qm", "before")
|
|
136
|
+
for path, text in case["after"].items():
|
|
137
|
+
(into / path).parent.mkdir(parents=True, exist_ok=True)
|
|
138
|
+
(into / path).write_text(text)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
async def review_case(case: dict[str, Any], model: LanguageModel, mode: str | None = None) -> ReviewReport:
|
|
142
|
+
settings = for_mode(mode or case.get("mode") or "standard")
|
|
143
|
+
with tempfile.TemporaryDirectory(prefix="mlai-eval-") as tmp:
|
|
144
|
+
root = Path(tmp)
|
|
145
|
+
checkout(case, root)
|
|
146
|
+
reader = LocalGitReader(root, scope="changes", title=case["name"])
|
|
147
|
+
publisher = EventPublisher(EventLog(None), root)
|
|
148
|
+
service = ReviewService(
|
|
149
|
+
context=ContextBuilder(reader, TreeSitterOutliner()),
|
|
150
|
+
workspaces=CheckoutProvider(root),
|
|
151
|
+
publisher=publisher,
|
|
152
|
+
reviewer=reviewer_for(model, ModelSettings("", ""), settings), # type: ignore[arg-type]
|
|
153
|
+
limits=ReviewLimits(),
|
|
154
|
+
)
|
|
155
|
+
from mlai_review import pipeline
|
|
156
|
+
|
|
157
|
+
head, base = await reader.head(_target())
|
|
158
|
+
return await pipeline.review(service, _target(head, base))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _target(head: str = "", base: str = "") -> Any:
|
|
162
|
+
from mlai_review.contracts.review import ReviewPullRequestInput
|
|
163
|
+
|
|
164
|
+
return ReviewPullRequestInput(0, 0, "local/eval", 0, head or "0" * 40, base or "0" * 40, "eval")
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def load_cases(directory: Path | None = None) -> list[dict[str, Any]]:
|
|
168
|
+
folder = directory or CASES
|
|
169
|
+
return [json.loads(p.read_text()) for p in sorted(folder.glob("*.json")) if p.name != "replies.json"]
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
async def evaluate(
|
|
173
|
+
cases: list[dict[str, Any]], model: LanguageModel, mode: str | None = None
|
|
174
|
+
) -> Score:
|
|
175
|
+
results = []
|
|
176
|
+
for case in cases:
|
|
177
|
+
report = await review_case(case, model, mode)
|
|
178
|
+
results.append(score_case(case, report))
|
|
179
|
+
return Score(tuple(results))
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def render(score: Score) -> str:
|
|
183
|
+
lines = ["| Case | Expected | Found | Published | Correct |", "|---|---|---|---|---|"]
|
|
184
|
+
lines += [f"| {c.name} | {c.expected} | {c.found} | {c.published} | {c.correct} |" for c in score.cases]
|
|
185
|
+
lines += ["", f"Recall {score.recall:.0%}, precision {score.precision:.0%}"]
|
|
186
|
+
return "\n".join(lines)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def main(args: argparse.Namespace) -> int:
|
|
190
|
+
import os
|
|
191
|
+
|
|
192
|
+
folder = args.cases or CASES
|
|
193
|
+
replies_file = folder / "replies.json"
|
|
194
|
+
replies = json.loads(replies_file.read_text()) if replies_file.is_file() else {}
|
|
195
|
+
cases = load_cases(folder)
|
|
196
|
+
|
|
197
|
+
async def go() -> Score:
|
|
198
|
+
if not args.record:
|
|
199
|
+
return await evaluate(cases, ReplayModel(replies), args.mode)
|
|
200
|
+
from contextlib import AsyncExitStack
|
|
201
|
+
|
|
202
|
+
from mlai_review.__main__ import model_settings
|
|
203
|
+
from mlai_review.runtime import model_for
|
|
204
|
+
|
|
205
|
+
request = json.loads(args.request.read_text())
|
|
206
|
+
settings = model_settings(request.get("model") or {})
|
|
207
|
+
async with AsyncExitStack() as resources:
|
|
208
|
+
inner = await model_for(
|
|
209
|
+
settings,
|
|
210
|
+
os.environ.pop("MODEL_API_KEY", ""),
|
|
211
|
+
resources,
|
|
212
|
+
for_mode(args.mode or "standard"),
|
|
213
|
+
)
|
|
214
|
+
recorder = RecordingModel(inner, replies)
|
|
215
|
+
score = await evaluate(cases, recorder, args.mode)
|
|
216
|
+
replies_file.write_text(json.dumps(recorder.replies, indent=1, sort_keys=True))
|
|
217
|
+
return score
|
|
218
|
+
|
|
219
|
+
score = asyncio.run(go())
|
|
220
|
+
print(render(score))
|
|
221
|
+
return 0
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "missing-ownership-check",
|
|
3
|
+
"mode": "standard",
|
|
4
|
+
"before": {
|
|
5
|
+
"shop/orders.py": "def cancel_order(db, user, order_id):\n order = db.orders.get(order_id)\n if order is None or order.user_id != user.id:\n raise PermissionError(order_id)\n order.status = \"cancelled\"\n db.save(order)\n return order\n"
|
|
6
|
+
},
|
|
7
|
+
"after": {
|
|
8
|
+
"shop/orders.py": "def cancel_order(db, user, order_id):\n order = db.orders.get(order_id)\n if order is None:\n raise LookupError(order_id)\n order.status = \"cancelled\"\n db.save(order)\n return order\n"
|
|
9
|
+
},
|
|
10
|
+
"expected": [
|
|
11
|
+
{
|
|
12
|
+
"path": "shop/orders.py",
|
|
13
|
+
"line": 3,
|
|
14
|
+
"keywords": [
|
|
15
|
+
"owner",
|
|
16
|
+
"authoriz",
|
|
17
|
+
"permission",
|
|
18
|
+
"any user"
|
|
19
|
+
]
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "off-by-one",
|
|
3
|
+
"mode": "standard",
|
|
4
|
+
"before": {
|
|
5
|
+
"src/page.ts": "export function page<T>(items: T[], size: number, n: number): T[] {\n const start = n * size;\n return items.slice(start, start + size);\n}\n"
|
|
6
|
+
},
|
|
7
|
+
"after": {
|
|
8
|
+
"src/page.ts": "export function page<T>(items: T[], size: number, n: number): T[] {\n const start = n * size;\n return items.slice(start, start + size - 1);\n}\n"
|
|
9
|
+
},
|
|
10
|
+
"expected": [
|
|
11
|
+
{
|
|
12
|
+
"path": "src/page.ts",
|
|
13
|
+
"line": 3,
|
|
14
|
+
"keywords": [
|
|
15
|
+
"off-by-one",
|
|
16
|
+
"one item",
|
|
17
|
+
"drops",
|
|
18
|
+
"last"
|
|
19
|
+
]
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "sql-injection",
|
|
3
|
+
"mode": "security",
|
|
4
|
+
"before": {
|
|
5
|
+
"app/users.py": "import sqlite3\n\n\ndef find_user(db, name):\n cur = db.cursor()\n cur.execute(\"SELECT id, name FROM users WHERE name = ?\", (name,))\n return cur.fetchone()\n"
|
|
6
|
+
},
|
|
7
|
+
"after": {
|
|
8
|
+
"app/users.py": "import sqlite3\n\n\ndef find_user(db, name):\n cur = db.cursor()\n cur.execute(f\"SELECT id, name FROM users WHERE name = '{name}'\")\n return cur.fetchone()\n"
|
|
9
|
+
},
|
|
10
|
+
"expected": [
|
|
11
|
+
{
|
|
12
|
+
"path": "app/users.py",
|
|
13
|
+
"line": 6,
|
|
14
|
+
"keywords": [
|
|
15
|
+
"sql",
|
|
16
|
+
"injection"
|
|
17
|
+
]
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""How ML AI Review's comments were received in a repository: useful, noise, no signal.
|
|
2
|
+
|
|
3
|
+
GITHUB_TOKEN=... python -m mlai_review feedback owner/name [owner/name ...] --days 14
|
|
4
|
+
|
|
5
|
+
Prints Markdown.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
from mlai_review.domain.feedback import FeedbackSummary, FeedbackVerdict
|
|
12
|
+
|
|
13
|
+
_MARK = {True: "met", False: "missed", None: "pending"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def render(full_name: str, days: int, summary: FeedbackSummary) -> str:
|
|
17
|
+
useful = summary.count(FeedbackVerdict.USEFUL)
|
|
18
|
+
noise = summary.count(FeedbackVerdict.NOISE)
|
|
19
|
+
unknown = summary.count(FeedbackVerdict.UNKNOWN)
|
|
20
|
+
median = summary.review_seconds(0.5)
|
|
21
|
+
p90 = summary.review_seconds(0.9)
|
|
22
|
+
lines = [
|
|
23
|
+
f"## {full_name}, last {days} days",
|
|
24
|
+
"",
|
|
25
|
+
f"- Pull requests reviewed: {summary.pull_requests}",
|
|
26
|
+
f"- Reviews: {len(summary.runs)} ({summary.failed_reviews} could not complete)",
|
|
27
|
+
"- Review time: "
|
|
28
|
+
+ (
|
|
29
|
+
"n/a"
|
|
30
|
+
if median is None
|
|
31
|
+
else f"median {median / 60:.1f} min, p90 {(p90 or 0) / 60:.1f} min"
|
|
32
|
+
),
|
|
33
|
+
f"- Comments: {len(summary.comments)} "
|
|
34
|
+
f"(useful {useful}, noise {noise}, no signal {unknown})",
|
|
35
|
+
"- Useful rate by kind: "
|
|
36
|
+
+ ", ".join(
|
|
37
|
+
f"{label} {'n/a' if rate is None else f'{rate:.0%}'}"
|
|
38
|
+
for label, rate in (
|
|
39
|
+
("stated", summary.useful_rate_of(tentative=False)),
|
|
40
|
+
("possible", summary.useful_rate_of(tentative=True)),
|
|
41
|
+
)
|
|
42
|
+
),
|
|
43
|
+
"",
|
|
44
|
+
"| Trial bar | Target | Actual | |",
|
|
45
|
+
"|---|---|---|---|",
|
|
46
|
+
]
|
|
47
|
+
lines += [f"| {c.name} | {c.target} | {c.actual} | {_MARK[c.met]} |" for c in summary.trial()]
|
|
48
|
+
if summary.comments:
|
|
49
|
+
lines += ["", "| PR | Comment | Verdict | Signal |", "|---|---|---|---|"]
|
|
50
|
+
for judged in sorted(summary.comments, key=lambda j: j.comment.created_at, reverse=True):
|
|
51
|
+
c = judged.comment
|
|
52
|
+
title = c.title.replace("|", "\\|")[:90]
|
|
53
|
+
kind = " (possible)" if c.tentative else ""
|
|
54
|
+
lines.append(
|
|
55
|
+
f"| #{c.pull_number} | `{c.path}`: {title}{kind} | {judged.verdict.value} | "
|
|
56
|
+
f"{judged.signal.value} |"
|
|
57
|
+
)
|
|
58
|
+
return "\n".join(lines) + "\n"
|
|
59
|
+
|
|
60
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Adapters that implement the application ports against real systems."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Static-analysis adapters: where the repository's own tools get to run."""
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""No static analysis: the review runs on the model alone."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from mlai_review.contracts.review import ReviewPullRequestInput
|
|
6
|
+
from mlai_review.application.ports import StaticAnalyzer
|
|
7
|
+
from mlai_review.domain.diagnostics import Diagnostic
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class NoStaticAnalysis(StaticAnalyzer):
|
|
11
|
+
async def analyse(self, target: ReviewPullRequestInput) -> tuple[Diagnostic, ...]:
|
|
12
|
+
return ()
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""From the sandbox's wire format to the domain's diagnostics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from mlai_review.contracts.analysis import AnalysisResult
|
|
6
|
+
from mlai_review.domain.diagnostics import Diagnostic, DiagnosticLevel
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def diagnostics_from(result: AnalysisResult) -> tuple[Diagnostic, ...]:
|
|
10
|
+
out = []
|
|
11
|
+
for d in result.diagnostics:
|
|
12
|
+
try:
|
|
13
|
+
level = DiagnosticLevel(d.level)
|
|
14
|
+
except ValueError:
|
|
15
|
+
level = DiagnosticLevel.WARNING # an unknown level is never promoted to an error
|
|
16
|
+
out.append(Diagnostic(d.tool, d.path, d.line, d.rule, d.message, level))
|
|
17
|
+
return tuple(out)
|
|
File without changes
|