mlai-code 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/checkpoints.mjs +217 -0
- package/src/cli.mjs +178 -31
- package/src/cloud.mjs +2 -0
- package/src/engine.mjs +1 -1
- package/src/headless.mjs +25 -9
- package/src/home.mjs +3 -2
- package/src/host.mjs +141 -13
- package/src/ide-quick.mjs +147 -0
- package/src/ide.mjs +182 -0
- package/src/netbox.mjs +1 -1
- package/src/parity.mjs +20 -1
- package/src/relay.mjs +1 -1
- package/src/render.mjs +12 -3
- package/src/runtime.mjs +23 -1
- package/src/sandbox.mjs +19 -3
- package/src/server.mjs +46 -7
- package/src/session.mjs +53 -10
- package/src/shared-data.mjs +1 -1
- package/src/tui/acp.mjs +121 -19
- package/src/tui/app.mjs +24 -5
- package/src/tui/chat.mjs +43 -16
- package/src/tui/cmd-config.mjs +3 -1
- package/src/tui/cmd-core.mjs +11 -7
- package/src/tui/cmd-extensibility.mjs +13 -2
- package/src/tui/commands.mjs +2 -1
- package/src/tui/extensions-local.mjs +2 -0
- package/src/tui/mcp-config.mjs +22 -4
- package/src/tui/review-cli.mjs +15 -4
- package/src/tui/snapshots.mjs +57 -16
- package/src/tui/statusline.mjs +2 -1
- package/src/two-words.mjs +14 -8
- package/src/view-sync.mjs +48 -0
- package/src/wakeups.mjs +2 -2
- package/src/winenv.mjs +1 -1
- package/src/workspace-api.mjs +21 -4
- package/vendor/client-sdk/index.cjs +5 -4
- package/vendor/cloud-cli/mlai.mjs +184 -28
- package/worker/architecture_worker.py +33 -4
- package/worker/browser_tool.py +19 -1
- package/worker/build_planner.py +75 -7
- package/worker/chat_fork.py +4 -1
- package/worker/checkpoints.py +71 -8
- package/worker/cloud_pty.py +91 -2
- package/worker/command_safety.py +80 -3
- package/worker/compaction_brief.py +186 -0
- package/worker/connected_turn.py +30 -0
- package/worker/context_manager.py +6 -0
- package/worker/conversation.py +139 -12
- package/worker/deck_render.js +11 -0
- package/worker/deck_tools.py +825 -0
- package/worker/git_sync.py +42 -17
- package/worker/goals.py +4 -0
- package/worker/image_generation.py +102 -23
- package/worker/kimi_bridge.py +580 -52
- package/worker/kimi_gates.py +74 -6
- package/worker/kimi_hook.py +213 -19
- package/worker/media_tools.py +182 -0
- package/worker/memory.py +2 -2
- package/worker/mlai_hooks.py +314 -5
- package/worker/mlai_mcp.py +49 -11
- package/worker/mlai_review/README.md +35 -0
- package/worker/mlai_review/__init__.py +1 -0
- package/worker/mlai_review/__main__.py +261 -0
- package/worker/mlai_review/_compat.py +29 -0
- package/worker/mlai_review/analysis/__init__.py +3 -0
- package/worker/mlai_review/analysis/detect.py +50 -0
- package/worker/mlai_review/analysis/main.py +128 -0
- package/worker/mlai_review/analysis/messages.py +8 -0
- package/worker/mlai_review/analysis/tools.py +194 -0
- package/worker/mlai_review/application/__init__.py +1 -0
- package/worker/mlai_review/application/asking.py +53 -0
- package/worker/mlai_review/application/challengers.py +142 -0
- package/worker/mlai_review/application/context_builder.py +194 -0
- package/worker/mlai_review/application/errors.py +17 -0
- package/worker/mlai_review/application/feedback.py +52 -0
- package/worker/mlai_review/application/matching.py +46 -0
- package/worker/mlai_review/application/messages.py +41 -0
- package/worker/mlai_review/application/ports.py +222 -0
- package/worker/mlai_review/application/prompts.py +342 -0
- package/worker/mlai_review/application/review_service.py +200 -0
- package/worker/mlai_review/application/reviewers.py +438 -0
- package/worker/mlai_review/application/streaming.py +91 -0
- package/worker/mlai_review/application/tools.py +138 -0
- package/worker/mlai_review/application/usage.py +145 -0
- package/worker/mlai_review/application/write_check.py +46 -0
- package/worker/mlai_review/contracts/__init__.py +5 -0
- package/worker/mlai_review/contracts/analysis.py +57 -0
- package/worker/mlai_review/contracts/review.py +42 -0
- package/worker/mlai_review/domain/__init__.py +4 -0
- package/worker/mlai_review/domain/changes.py +77 -0
- package/worker/mlai_review/domain/context.py +147 -0
- package/worker/mlai_review/domain/diagnostics.py +239 -0
- package/worker/mlai_review/domain/feedback.py +216 -0
- package/worker/mlai_review/domain/findings.py +505 -0
- package/worker/mlai_review/domain/lifecycle.py +85 -0
- package/worker/mlai_review/domain/partition.py +42 -0
- package/worker/mlai_review/domain/relations.py +166 -0
- package/worker/mlai_review/domain/relevance.py +202 -0
- package/worker/mlai_review/domain/report.py +99 -0
- package/worker/mlai_review/domain/structure.py +56 -0
- package/worker/mlai_review/domain/writes.py +86 -0
- package/worker/mlai_review/eval.py +221 -0
- package/worker/mlai_review/eval_cases/clean-rename.json +11 -0
- package/worker/mlai_review/eval_cases/missing-ownership-check.json +22 -0
- package/worker/mlai_review/eval_cases/off-by-one.json +22 -0
- package/worker/mlai_review/eval_cases/sql-injection.json +20 -0
- package/worker/mlai_review/feedback_report.py +60 -0
- package/worker/mlai_review/infrastructure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/none.py +12 -0
- package/worker/mlai_review/infrastructure/analysis/translate.py +17 -0
- package/worker/mlai_review/infrastructure/github/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/github/client.py +445 -0
- package/worker/mlai_review/infrastructure/github/feedback.py +212 -0
- package/worker/mlai_review/infrastructure/github/http.py +31 -0
- package/worker/mlai_review/infrastructure/github/rendering.py +398 -0
- package/worker/mlai_review/infrastructure/github/token.py +20 -0
- package/worker/mlai_review/infrastructure/local/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/local/analyzer.py +48 -0
- package/worker/mlai_review/infrastructure/local/events.py +100 -0
- package/worker/mlai_review/infrastructure/local/git_reader.py +233 -0
- package/worker/mlai_review/infrastructure/local/publisher.py +138 -0
- package/worker/mlai_review/infrastructure/messages.py +54 -0
- package/worker/mlai_review/infrastructure/models/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/models/openai_compatible.py +269 -0
- package/worker/mlai_review/infrastructure/structure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/structure/heuristic.py +186 -0
- package/worker/mlai_review/infrastructure/structure/tree_sitter.py +186 -0
- package/worker/mlai_review/infrastructure/workspace/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/workspace/local.py +105 -0
- package/worker/mlai_review/infrastructure/workspace/memory.py +77 -0
- package/worker/mlai_review/infrastructure/workspace/paths.py +17 -0
- package/worker/mlai_review/pipeline.py +89 -0
- package/worker/mlai_review/runtime.py +194 -0
- package/worker/mlai_review/settings.py +139 -0
- package/worker/model_stream.py +21 -7
- package/worker/offline_guard.py +1 -1
- package/worker/preview.py +62 -4
- package/worker/product_facts.py +20 -0
- package/worker/project_context.py +33 -7
- package/worker/reply_language.py +92 -10
- package/worker/repo_policy.py +5 -0
- package/worker/roles.json +199 -37
- package/worker/roles.py +271 -9
- package/worker/runner.py +366 -53
- package/worker/runtime_control.py +64 -3
- package/worker/secret_scan.py +3 -2
- package/worker/server_memory.py +122 -0
- package/worker/site_check.py +279 -0
- package/worker/skill_library.py +15 -3
- package/worker/skill_pack.py +170 -16
- package/worker/subagents.py +6 -2
- package/worker/tool_catalog.py +4 -3
- package/worker/verification.py +156 -19
- package/worker/wakeups.py +56 -4
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Gathers what a reviewer sees, within a size budget."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
from mlai_review.contracts.review import ReviewPullRequestInput
|
|
9
|
+
from mlai_review.application import messages
|
|
10
|
+
from mlai_review.application.ports import GrepMatch, Outliner, PullRequestReader, Workspace
|
|
11
|
+
from mlai_review.domain.changes import changed_lines, parse_patch
|
|
12
|
+
from mlai_review.domain.context import FileView, ReviewContext
|
|
13
|
+
from mlai_review.domain.diagnostics import Diagnostic, on_changed_lines
|
|
14
|
+
from mlai_review.domain.relations import (
|
|
15
|
+
Relation,
|
|
16
|
+
RelationKind,
|
|
17
|
+
called_names,
|
|
18
|
+
changed_definitions,
|
|
19
|
+
definition_pattern,
|
|
20
|
+
)
|
|
21
|
+
from mlai_review.domain.relevance import Triage, is_source, is_test, triage
|
|
22
|
+
|
|
23
|
+
_log = logging.getLogger("mlai.orchestrator.context")
|
|
24
|
+
|
|
25
|
+
# Budgets for what the reviewer is shown. A file whose view is over the per-file limit is
|
|
26
|
+
# reviewed from its diff only; once the total is spent, remaining files are too.
|
|
27
|
+
MAX_FILE_CHARS = 60_000
|
|
28
|
+
MAX_TOTAL_CHARS = 150_000
|
|
29
|
+
# Files larger than this are not read at all.
|
|
30
|
+
MAX_SOURCE_BYTES = 2_000_000
|
|
31
|
+
# Related code: how many names to look up, matches to scan, and lines to show.
|
|
32
|
+
MAX_LOOKUP_NAMES = 24
|
|
33
|
+
MAX_MATCHES = 4_000
|
|
34
|
+
# A name with more uses than this is too common to be informative; methods (get, handle,
|
|
35
|
+
# serialize...) share names across classes, so their bar is lower.
|
|
36
|
+
MAX_GENERIC_MATCHES = 60
|
|
37
|
+
MAX_GENERIC_METHOD_MATCHES = 15
|
|
38
|
+
MAX_DEFINITIONS_PER_NAME = 2
|
|
39
|
+
MAX_REFERENCES_PER_NAME = 4
|
|
40
|
+
MAX_RELATED_CHARS = 8_000
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ContextBuilder:
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
reader: PullRequestReader,
|
|
47
|
+
outliner: Outliner,
|
|
48
|
+
*,
|
|
49
|
+
max_file_chars: int = MAX_FILE_CHARS,
|
|
50
|
+
max_total_chars: int = MAX_TOTAL_CHARS,
|
|
51
|
+
) -> None:
|
|
52
|
+
self._reader = reader
|
|
53
|
+
self._outliner = outliner
|
|
54
|
+
self._max_file_chars = max_file_chars
|
|
55
|
+
self._max_total_chars = max_total_chars
|
|
56
|
+
|
|
57
|
+
async def head(self, target: ReviewPullRequestInput) -> tuple[str, str]:
|
|
58
|
+
return await self._reader.head(target)
|
|
59
|
+
|
|
60
|
+
async def changes(self, target: ReviewPullRequestInput, *, since: str | None = None) -> Triage:
|
|
61
|
+
"""What the pull request changed (or only what changed since a commit), with noise
|
|
62
|
+
files set aside."""
|
|
63
|
+
result = triage(await self._reader.changed_files(target, since=since))
|
|
64
|
+
if result.skipped:
|
|
65
|
+
_log.info(messages.FILES_SKIPPED, len(result.skipped), ", ".join(result.skipped))
|
|
66
|
+
return result
|
|
67
|
+
|
|
68
|
+
async def build(
|
|
69
|
+
self,
|
|
70
|
+
target: ReviewPullRequestInput,
|
|
71
|
+
changes: Triage,
|
|
72
|
+
workspace: Workspace | None,
|
|
73
|
+
diagnostics: tuple[Diagnostic, ...] = (),
|
|
74
|
+
) -> ReviewContext:
|
|
75
|
+
"""The reviewer's view of the change. Without a workspace, it is the diffs alone."""
|
|
76
|
+
info = await self._reader.pull_request(target)
|
|
77
|
+
views: dict[str, FileView] = {}
|
|
78
|
+
if workspace is not None:
|
|
79
|
+
budget = self._max_total_chars
|
|
80
|
+
for change in changes.reviewable.files:
|
|
81
|
+
if change.status == "removed":
|
|
82
|
+
continue
|
|
83
|
+
text = await workspace.read(change.path, max_bytes=MAX_SOURCE_BYTES)
|
|
84
|
+
if text is None:
|
|
85
|
+
continue
|
|
86
|
+
view = FileView.focus(text, change.patch, self._outliner.outline(change.path, text))
|
|
87
|
+
if view.chars > self._max_file_chars or view.chars > budget:
|
|
88
|
+
continue
|
|
89
|
+
views[change.path] = view
|
|
90
|
+
budget -= view.chars
|
|
91
|
+
related = await self._related(workspace, changes, views) if workspace else ()
|
|
92
|
+
return ReviewContext(
|
|
93
|
+
pull_request=info,
|
|
94
|
+
changes=changes.reviewable,
|
|
95
|
+
views=views,
|
|
96
|
+
contents_available=workspace is not None,
|
|
97
|
+
skipped=changes.skipped,
|
|
98
|
+
related=related,
|
|
99
|
+
diagnostics=on_changed_lines(diagnostics, changes.reviewable),
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
async def _related(
|
|
103
|
+
self, workspace: Workspace, changes: Triage, views: dict[str, FileView]
|
|
104
|
+
) -> tuple[Relation, ...]:
|
|
105
|
+
"""Definitions of what the change calls, and uses of what it changes, from one search
|
|
106
|
+
over the checkout. Test files are reviewed on their own and not expanded."""
|
|
107
|
+
callees: list[str] = []
|
|
108
|
+
touched: dict[str, str] = {} # changed definition -> file it lives in
|
|
109
|
+
methods: set[str] = set()
|
|
110
|
+
for change in changes.reviewable.files:
|
|
111
|
+
view = views.get(change.path)
|
|
112
|
+
if view is None or view.outline is None or is_test(change.path):
|
|
113
|
+
continue
|
|
114
|
+
defined_here = {d.name for d in view.outline.definitions}
|
|
115
|
+
methods |= {d.name for d in view.outline.definitions if d.depth > 0}
|
|
116
|
+
for name in changed_definitions(view.outline, changed_lines(change.patch)):
|
|
117
|
+
touched.setdefault(name, change.path)
|
|
118
|
+
added = [d.text for d in parse_patch(change.patch) if d.kind == "+"]
|
|
119
|
+
for name in called_names(added, defined_here=defined_here):
|
|
120
|
+
if name not in callees and name not in touched:
|
|
121
|
+
callees.append(name)
|
|
122
|
+
names = [*touched, *callees][:MAX_LOOKUP_NAMES]
|
|
123
|
+
if not names:
|
|
124
|
+
return ()
|
|
125
|
+
pattern = r"\b(?:" + "|".join(re.escape(n) for n in names) + r")\b"
|
|
126
|
+
matches = await workspace.grep(pattern, limit=MAX_MATCHES)
|
|
127
|
+
return _select_related(matches, touched, callees, set(views), methods)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _select_related(
|
|
131
|
+
matches: tuple[GrepMatch, ...],
|
|
132
|
+
touched: dict[str, str],
|
|
133
|
+
callees: list[str],
|
|
134
|
+
changed_paths: set[str],
|
|
135
|
+
methods: frozenset[str] | set[str] = frozenset(),
|
|
136
|
+
) -> tuple[Relation, ...]:
|
|
137
|
+
by_name: dict[str, list[GrepMatch]] = {}
|
|
138
|
+
for match in matches:
|
|
139
|
+
if not is_source(match.path) or match.text.lstrip()[:1] in ("#", "/", "*"):
|
|
140
|
+
continue # not code, or a comment
|
|
141
|
+
for name in (*touched, *callees):
|
|
142
|
+
if re.search(rf"\b{re.escape(name)}\b", match.text):
|
|
143
|
+
by_name.setdefault(name, []).append(match)
|
|
144
|
+
|
|
145
|
+
def too_common(name: str) -> bool:
|
|
146
|
+
limit = MAX_GENERIC_METHOD_MATCHES if name in methods else MAX_GENERIC_MATCHES
|
|
147
|
+
return len(by_name.get(name, [])) > limit
|
|
148
|
+
|
|
149
|
+
related: list[Relation] = []
|
|
150
|
+
for name in callees:
|
|
151
|
+
if too_common(name):
|
|
152
|
+
continue
|
|
153
|
+
definitions = [
|
|
154
|
+
m
|
|
155
|
+
for m in by_name.get(name, [])
|
|
156
|
+
if m.path not in changed_paths and definition_pattern(name).search(m.text)
|
|
157
|
+
]
|
|
158
|
+
for match in definitions[:MAX_DEFINITIONS_PER_NAME]:
|
|
159
|
+
related.append(
|
|
160
|
+
Relation(RelationKind.DEFINITION, name, match.path, match.line, match.text)
|
|
161
|
+
)
|
|
162
|
+
for name, origin in touched.items():
|
|
163
|
+
if too_common(name):
|
|
164
|
+
continue
|
|
165
|
+
# Uses only, in the same language: another class's method of the same name, or a
|
|
166
|
+
# same-named type in the frontend, is not a use of ours.
|
|
167
|
+
uses = [
|
|
168
|
+
m
|
|
169
|
+
for m in by_name.get(name, [])
|
|
170
|
+
if m.path != origin
|
|
171
|
+
and _suffix(m.path) == _suffix(origin)
|
|
172
|
+
and not definition_pattern(name).search(m.text)
|
|
173
|
+
]
|
|
174
|
+
uses.sort(key=lambda m: (is_test(m.path), m.path, m.line)) # code first, then tests
|
|
175
|
+
for match in uses[:MAX_REFERENCES_PER_NAME]:
|
|
176
|
+
related.append(
|
|
177
|
+
Relation(RelationKind.REFERENCE, name, match.path, match.line, match.text)
|
|
178
|
+
)
|
|
179
|
+
kept: list[Relation] = []
|
|
180
|
+
seen: set[tuple[str, int]] = set()
|
|
181
|
+
spent = 0
|
|
182
|
+
for relation in related:
|
|
183
|
+
if (relation.path, relation.line) in seen:
|
|
184
|
+
continue
|
|
185
|
+
spent += len(relation.text) + len(relation.path) + 8
|
|
186
|
+
if spent > MAX_RELATED_CHARS:
|
|
187
|
+
break
|
|
188
|
+
seen.add((relation.path, relation.line))
|
|
189
|
+
kept.append(relation)
|
|
190
|
+
return tuple(kept)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _suffix(path: str) -> str:
|
|
194
|
+
return path[path.rfind(".") :].lower() if "." in path else ""
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Errors that tell the workflow whether retrying could help."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ReviewError(Exception):
|
|
5
|
+
"""A step failed in a way that may succeed on retry (network, rate limit, 5xx)."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class PermanentReviewError(ReviewError):
|
|
9
|
+
"""A step failed in a way retrying cannot fix (missing permission, PR or repo gone)."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AccessDenied(PermanentReviewError):
|
|
13
|
+
"""GitHub refused the token (401, 403 or 404): it lacks a permission or cannot see the repository."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ForbiddenPath(Exception):
|
|
17
|
+
"""A read asked for a path outside the checkout. Refused, never retried."""
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Feedback on ML AI's comments: a report for the people running a trial, and the team's
|
|
2
|
+
dismissals, which every later review of the repository takes into account."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
from datetime import datetime, timedelta
|
|
8
|
+
|
|
9
|
+
from mlai_review.application import messages
|
|
10
|
+
from mlai_review.application.errors import ReviewError
|
|
11
|
+
from mlai_review.application.ports import FeedbackSource, Repository
|
|
12
|
+
from mlai_review.domain.feedback import (
|
|
13
|
+
Dismissal,
|
|
14
|
+
FeedbackSummary,
|
|
15
|
+
dismissals,
|
|
16
|
+
judge,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
_log = logging.getLogger("mlai.orchestrator.feedback")
|
|
20
|
+
|
|
21
|
+
# How far back a review looks for findings the team dismissed, and how many it keeps.
|
|
22
|
+
DISMISSALS_LOOKBACK = timedelta(days=90)
|
|
23
|
+
MAX_DISMISSALS = 20
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class FeedbackReporter:
|
|
27
|
+
def __init__(self, source: FeedbackSource) -> None:
|
|
28
|
+
self._source = source
|
|
29
|
+
|
|
30
|
+
async def report(self, full_name: str, since: datetime) -> FeedbackSummary:
|
|
31
|
+
repository = await self._source.locate(full_name)
|
|
32
|
+
comments = await self._source.posted_comments(repository, since)
|
|
33
|
+
runs = await self._source.review_runs(repository, since)
|
|
34
|
+
return FeedbackSummary(comments=tuple(judge(c) for c in comments), runs=runs)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class TeamFeedback:
|
|
38
|
+
"""Findings the team dismissed in this repository, for a review about to run. Best
|
|
39
|
+
effort: if feedback cannot be read, the review goes ahead without it."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, source: FeedbackSource, clock: type[datetime] = datetime) -> None:
|
|
42
|
+
self._source = source
|
|
43
|
+
self._clock = clock
|
|
44
|
+
|
|
45
|
+
async def dismissed(self, repository: Repository) -> tuple[Dismissal, ...]:
|
|
46
|
+
since = self._clock.now().astimezone() - DISMISSALS_LOOKBACK
|
|
47
|
+
try:
|
|
48
|
+
comments = await self._source.posted_comments(repository, since, with_threads=False)
|
|
49
|
+
except ReviewError as exc:
|
|
50
|
+
_log.warning(messages.FEEDBACK_UNAVAILABLE, exc)
|
|
51
|
+
return ()
|
|
52
|
+
return dismissals(comments)[:MAX_DISMISSALS]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Whether two findings are one issue told in different words. Independent readings of a
|
|
2
|
+
change word the same defect differently, and nearby claims share as few words with each other
|
|
3
|
+
as distinct defects do, so words alone cannot tell; a short question to the model can."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
from abc import ABC, abstractmethod
|
|
9
|
+
|
|
10
|
+
from mlai_review.application import messages, prompts
|
|
11
|
+
from mlai_review.application.errors import ReviewError
|
|
12
|
+
from mlai_review.application.ports import LanguageModel
|
|
13
|
+
from mlai_review.domain.findings import Finding
|
|
14
|
+
|
|
15
|
+
_log = logging.getLogger("mlai.orchestrator.matcher")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class IssueMatcher(ABC):
|
|
19
|
+
@abstractmethod
|
|
20
|
+
async def same(self, a: Finding, b: Finding) -> bool:
|
|
21
|
+
"""True when both describe one defect: one fix resolves both."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class NoMatcher(IssueMatcher):
|
|
25
|
+
"""Words alone decide (see domain.findings.same_issue)."""
|
|
26
|
+
|
|
27
|
+
async def same(self, a: Finding, b: Finding) -> bool:
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class LlmIssueMatcher(IssueMatcher):
|
|
32
|
+
"""One small model call per pair. A call that fails keeps the two apart: a repeated
|
|
33
|
+
comment is noise, a lost finding is worse."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, model: LanguageModel) -> None:
|
|
36
|
+
self._model = model
|
|
37
|
+
|
|
38
|
+
async def same(self, a: Finding, b: Finding) -> bool:
|
|
39
|
+
try:
|
|
40
|
+
answer = await self._model.complete_json(
|
|
41
|
+
system=prompts.SAME_ISSUE_SYSTEM, user=prompts.same_issue_question(a, b)
|
|
42
|
+
)
|
|
43
|
+
except ReviewError as exc:
|
|
44
|
+
_log.warning(messages.MATCH_FAILED, a.title, b.title, exc)
|
|
45
|
+
return False
|
|
46
|
+
return answer.get("same") is True
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Wording for warnings the application layer logs, and for what the reviewer's tools answer.
|
|
2
|
+
Adapters own their own wording (see mlai_review.infrastructure.messages)."""
|
|
3
|
+
|
|
4
|
+
# Logged (%-style, for the logging module)
|
|
5
|
+
CONTENTS_UNAVAILABLE = "repository contents unavailable, reviewing diffs only: %s"
|
|
6
|
+
CHANGE_TOO_LARGE = "change too large to review: %d files, %d changed lines"
|
|
7
|
+
FILES_SKIPPED = "left out %d noise file(s): %s"
|
|
8
|
+
INCREMENTAL_REVIEW = "reviewing only the changes since %s, which was reviewed earlier"
|
|
9
|
+
REVIEW_OUTCOME = "review candidates=%d rejected=%d merged=%d refuted=%d published=%d suppressed=%d"
|
|
10
|
+
MATCH_FAILED = "could not compare %r with %r, keeping both: %s"
|
|
11
|
+
MATCHED_LOG = "issues compared=%d joined=%d"
|
|
12
|
+
PARTS_LOG = "reviewing in %d parts of %s files"
|
|
13
|
+
PART_FAILED = "a part of the change could not be reviewed, publishing the others: %s"
|
|
14
|
+
AGREEMENT_LOG = "readings=%d issues by votes %s"
|
|
15
|
+
PASSES_AT_LEAST_ONE = "a review needs at least one pass, got {passes}"
|
|
16
|
+
CHALLENGE_FAILED = "refuter failed on %r, leaving it unchallenged: %s"
|
|
17
|
+
CHALLENGE_OUTCOME = "refuter %s %r: %s"
|
|
18
|
+
SPECIALIST_FAILED = "specialist %s failed, publishing the others: %s"
|
|
19
|
+
WRITE_CHECK_FAILED = "write check failed, publishing the readings: %s"
|
|
20
|
+
RECONCILED = "findings new=%d repeated=%d resolved=%d"
|
|
21
|
+
FEEDBACK_UNAVAILABLE = "team feedback unavailable, reviewing without it: %s"
|
|
22
|
+
DISMISSED_BY_FEEDBACK = "left out %d finding(s) the team dismissed before"
|
|
23
|
+
ANALYSIS_TIMED_OUT = "static analysis not back after %.0fs, reviewing without it"
|
|
24
|
+
ANALYSIS_FAILED = "static analysis unavailable, reviewing without it: %s"
|
|
25
|
+
LOOKUPS_USED = "specialist %s used %d lookup(s), %d chars of results"
|
|
26
|
+
REVIEW_USAGE = (
|
|
27
|
+
"review usage repository=%s pull=%d head=%s model_calls=%d prompt_tokens=%d "
|
|
28
|
+
"cached_tokens=%d completion_tokens=%d"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# Answered to the model by its tools (str.format)
|
|
32
|
+
TOOL_UNKNOWN = "error: no tool named {name}"
|
|
33
|
+
TOOL_BAD_ARGUMENTS = "error: bad arguments for {name}: {reason}"
|
|
34
|
+
TOOL_BAD_PATTERN = "error: invalid regular expression: {reason}"
|
|
35
|
+
TOOL_REFUSED = "error: {reason}"
|
|
36
|
+
TOOL_NO_MATCHES = "no matches"
|
|
37
|
+
TOOL_MATCHES_CAPPED = "(more matches not shown; at most {limit} are returned, narrow the search)"
|
|
38
|
+
TOOL_FILE_UNREADABLE = "error: {path} does not exist or is not a text file"
|
|
39
|
+
TOOL_PAST_END = "error: {path} has only {total} lines"
|
|
40
|
+
TOOL_READ_RANGE = "(lines {start}-{end} of {total})"
|
|
41
|
+
TOOL_EMPTY_DIR = "{path} is empty or not a directory"
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""What the application needs from the outside world. Adapters live in infrastructure."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from contextlib import AbstractAsyncContextManager
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from mlai_review.contracts.review import ReviewPullRequestInput
|
|
12
|
+
from mlai_review.domain.changes import ChangeSet
|
|
13
|
+
from mlai_review.domain.context import PullRequestInfo
|
|
14
|
+
from mlai_review.domain.diagnostics import Diagnostic
|
|
15
|
+
from mlai_review.domain.feedback import PostedComment, ReviewRun
|
|
16
|
+
from mlai_review.domain.findings import Finding
|
|
17
|
+
from mlai_review.domain.lifecycle import PriorFinding
|
|
18
|
+
from mlai_review.domain.report import ReviewReport
|
|
19
|
+
from mlai_review.domain.structure import Outline
|
|
20
|
+
|
|
21
|
+
# A single read never returns more than this; larger files are treated as unreadable.
|
|
22
|
+
MAX_READ_BYTES = 2_000_000
|
|
23
|
+
MAX_GREP_MATCHES = 50
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class PullRequestReader(ABC):
|
|
27
|
+
@abstractmethod
|
|
28
|
+
async def pull_request(self, target: ReviewPullRequestInput) -> PullRequestInfo: ...
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
async def head(self, target: ReviewPullRequestInput) -> tuple[str, str]:
|
|
32
|
+
"""The pull request's head and base commits as they are now."""
|
|
33
|
+
|
|
34
|
+
@abstractmethod
|
|
35
|
+
async def changed_files(
|
|
36
|
+
self, target: ReviewPullRequestInput, *, since: str | None = None
|
|
37
|
+
) -> ChangeSet:
|
|
38
|
+
"""What the pull request changes; with `since`, only what changed between that commit
|
|
39
|
+
and the head. Returns None-equivalent (the full change) when `since` is not an
|
|
40
|
+
ancestor of the head, e.g. after a force push: callers check `ChangeSet.since`."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True, slots=True)
|
|
44
|
+
class GrepMatch:
|
|
45
|
+
path: str
|
|
46
|
+
line: int
|
|
47
|
+
text: str
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Workspace(ABC):
|
|
51
|
+
"""A read-only copy of the repository at the reviewed commit.
|
|
52
|
+
|
|
53
|
+
Reads never leave it: a path that is absolute, climbs with `..`, enters `.git`, or follows a
|
|
54
|
+
symlink out of the checkout raises ForbiddenPath. Paths are repository-relative, POSIX style.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
@abstractmethod
|
|
58
|
+
async def read(self, path: str, *, max_bytes: int = MAX_READ_BYTES) -> str | None:
|
|
59
|
+
"""Text of a file; None if it is missing, binary, or larger than max_bytes."""
|
|
60
|
+
|
|
61
|
+
@abstractmethod
|
|
62
|
+
async def grep(
|
|
63
|
+
self, pattern: str, *, glob: str | None = None, limit: int = MAX_GREP_MATCHES
|
|
64
|
+
) -> tuple[GrepMatch, ...]:
|
|
65
|
+
"""Lines matching a regular expression, at most `limit`, optionally only in files whose
|
|
66
|
+
path matches `glob`."""
|
|
67
|
+
|
|
68
|
+
@abstractmethod
|
|
69
|
+
async def list_dir(self, path: str = "") -> tuple[str, ...]:
|
|
70
|
+
"""Entries one level down, sorted, directories with a trailing slash."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class WorkspaceProvider(ABC):
|
|
74
|
+
@abstractmethod
|
|
75
|
+
def open(self, target: ReviewPullRequestInput) -> AbstractAsyncContextManager[Workspace]:
|
|
76
|
+
"""A workspace for the reviewed commit, deleted when the context exits.
|
|
77
|
+
|
|
78
|
+
Raises PermanentReviewError when the repository cannot be read at all (no permission,
|
|
79
|
+
repository gone) and ReviewError when a retry may succeed (network).
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class StaticAnalyzer(ABC):
|
|
84
|
+
"""Runs the repository's own kind of tools (compilers, linters) on the reviewed commit and
|
|
85
|
+
reports what they say. Those tools execute code from the pull request, so an adapter must
|
|
86
|
+
run them somewhere that holds no secrets and cannot reach anything that matters."""
|
|
87
|
+
|
|
88
|
+
@abstractmethod
|
|
89
|
+
async def analyse(self, target: ReviewPullRequestInput) -> tuple[Diagnostic, ...]:
|
|
90
|
+
"""Raise ReviewError when the analysis could not run; never raise for tool findings."""
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass(frozen=True, slots=True)
|
|
94
|
+
class Repository:
|
|
95
|
+
"""A repository the App is installed on, with what it takes to get a token for it."""
|
|
96
|
+
|
|
97
|
+
full_name: str
|
|
98
|
+
installation_id: int
|
|
99
|
+
repository_id: int
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def owner(self) -> str:
|
|
103
|
+
return self.full_name.split("/", 1)[0]
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def name(self) -> str:
|
|
107
|
+
return self.full_name.split("/", 1)[1]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class FeedbackSource(ABC):
|
|
111
|
+
"""What developers did with ML AI's comments, read back from where they work."""
|
|
112
|
+
|
|
113
|
+
@abstractmethod
|
|
114
|
+
async def locate(self, full_name: str) -> Repository:
|
|
115
|
+
"""The installation and id of a repository the App can see, by owner/name."""
|
|
116
|
+
|
|
117
|
+
@abstractmethod
|
|
118
|
+
async def posted_comments(
|
|
119
|
+
self, repository: Repository, since: datetime, *, with_threads: bool = True
|
|
120
|
+
) -> tuple[PostedComment, ...]:
|
|
121
|
+
"""ML AI's finding comments created since then, with reactions, people's replies and
|
|
122
|
+
(with_threads) whether each thread was resolved."""
|
|
123
|
+
|
|
124
|
+
@abstractmethod
|
|
125
|
+
async def review_runs(self, repository: Repository, since: datetime) -> tuple[ReviewRun, ...]:
|
|
126
|
+
"""Completed ML AI reviews of pull requests updated since then."""
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class Outliner(ABC):
|
|
130
|
+
"""Finds the definitions and imports of a source file."""
|
|
131
|
+
|
|
132
|
+
@abstractmethod
|
|
133
|
+
def outline(self, path: str, text: str) -> Outline | None:
|
|
134
|
+
"""None when the language is not supported."""
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class ReviewPublisher(ABC):
|
|
138
|
+
"""Shows review progress and results where the developer works (a GitHub check run)."""
|
|
139
|
+
|
|
140
|
+
@abstractmethod
|
|
141
|
+
async def start(self, target: ReviewPullRequestInput) -> int:
|
|
142
|
+
"""Show the review as in progress. Return an id for later updates."""
|
|
143
|
+
|
|
144
|
+
@abstractmethod
|
|
145
|
+
async def acknowledge(self, target: ReviewPullRequestInput) -> None:
|
|
146
|
+
"""Let whoever asked for the review in a comment see it was picked up."""
|
|
147
|
+
|
|
148
|
+
@abstractmethod
|
|
149
|
+
async def post_finding(self, target: ReviewPullRequestInput, finding: Finding) -> None:
|
|
150
|
+
"""Show one finding where the developer works, as soon as it is confirmed."""
|
|
151
|
+
|
|
152
|
+
@abstractmethod
|
|
153
|
+
async def complete(
|
|
154
|
+
self, target: ReviewPullRequestInput, publication_id: int, report: ReviewReport
|
|
155
|
+
) -> None:
|
|
156
|
+
"""Close the review: its summary, and replies on earlier findings it resolved."""
|
|
157
|
+
|
|
158
|
+
@abstractmethod
|
|
159
|
+
async def reviewed(self, target: ReviewPullRequestInput, sha: str) -> bool:
|
|
160
|
+
"""Whether a completed review of ours exists for that commit."""
|
|
161
|
+
|
|
162
|
+
async def last_reviewed(self, target: ReviewPullRequestInput) -> str | None:
|
|
163
|
+
"""The last head commit a completed review of ours judged on this pull request, when
|
|
164
|
+
the surface records one (the summary comment does); None otherwise."""
|
|
165
|
+
return None
|
|
166
|
+
|
|
167
|
+
@abstractmethod
|
|
168
|
+
async def prior_findings(self, target: ReviewPullRequestInput) -> tuple[PriorFinding, ...]:
|
|
169
|
+
"""Findings earlier reviews commented on this pull request, with whether each comment's
|
|
170
|
+
lines have changed since."""
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@dataclass(frozen=True, slots=True)
|
|
174
|
+
class ToolSpec:
|
|
175
|
+
"""A function the model may call, described to it in JSON Schema."""
|
|
176
|
+
|
|
177
|
+
name: str
|
|
178
|
+
description: str
|
|
179
|
+
parameters: dict[str, Any]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@dataclass(frozen=True, slots=True)
|
|
183
|
+
class ToolCall:
|
|
184
|
+
id: str
|
|
185
|
+
name: str
|
|
186
|
+
arguments: dict[str, Any]
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@dataclass(frozen=True, slots=True)
|
|
190
|
+
class Message:
|
|
191
|
+
"""One entry of a conversation with the model."""
|
|
192
|
+
|
|
193
|
+
role: str # system, user, assistant, tool
|
|
194
|
+
content: str = ""
|
|
195
|
+
tool_calls: tuple[ToolCall, ...] = () # assistant turns that asked for tools
|
|
196
|
+
tool_call_id: str | None = None # tool results, naming the call they answer
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
@dataclass(frozen=True, slots=True)
|
|
200
|
+
class Turn:
|
|
201
|
+
"""What the model said next: tool calls to run, or its answer as one JSON object. Without
|
|
202
|
+
tools on offer a turn always carries an answer; with tools, the model may instead reply in
|
|
203
|
+
plain text, which comes back as `text` for the caller to steer from."""
|
|
204
|
+
|
|
205
|
+
tool_calls: tuple[ToolCall, ...] = ()
|
|
206
|
+
answer: dict[str, Any] | None = None
|
|
207
|
+
text: str = ""
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class LanguageModel(ABC):
|
|
211
|
+
"""A chat model, one turn at a time. Which model and provider is an adapter's concern;
|
|
212
|
+
routing between models belongs behind this port too."""
|
|
213
|
+
|
|
214
|
+
@abstractmethod
|
|
215
|
+
async def next_turn(self, messages: tuple[Message, ...], tools: tuple[ToolSpec, ...]) -> Turn:
|
|
216
|
+
"""Raise ReviewError if the call can be retried, PermanentReviewError if not."""
|
|
217
|
+
|
|
218
|
+
async def complete_json(self, *, system: str, user: str) -> dict[str, Any]:
|
|
219
|
+
"""One question, one JSON answer, no tools."""
|
|
220
|
+
turn = await self.next_turn((Message("system", system), Message("user", user)), ())
|
|
221
|
+
assert turn.answer is not None
|
|
222
|
+
return turn.answer
|