mlai-code 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/checkpoints.mjs +217 -0
- package/src/cli.mjs +178 -31
- package/src/cloud.mjs +2 -0
- package/src/engine.mjs +1 -1
- package/src/headless.mjs +25 -9
- package/src/home.mjs +3 -2
- package/src/host.mjs +141 -13
- package/src/ide-quick.mjs +147 -0
- package/src/ide.mjs +182 -0
- package/src/netbox.mjs +1 -1
- package/src/parity.mjs +20 -1
- package/src/relay.mjs +1 -1
- package/src/render.mjs +12 -3
- package/src/runtime.mjs +23 -1
- package/src/sandbox.mjs +19 -3
- package/src/server.mjs +46 -7
- package/src/session.mjs +53 -10
- package/src/shared-data.mjs +1 -1
- package/src/tui/acp.mjs +121 -19
- package/src/tui/app.mjs +24 -5
- package/src/tui/chat.mjs +43 -16
- package/src/tui/cmd-config.mjs +3 -1
- package/src/tui/cmd-core.mjs +11 -7
- package/src/tui/cmd-extensibility.mjs +13 -2
- package/src/tui/commands.mjs +2 -1
- package/src/tui/extensions-local.mjs +2 -0
- package/src/tui/mcp-config.mjs +22 -4
- package/src/tui/review-cli.mjs +15 -4
- package/src/tui/snapshots.mjs +57 -16
- package/src/tui/statusline.mjs +2 -1
- package/src/two-words.mjs +14 -8
- package/src/view-sync.mjs +48 -0
- package/src/wakeups.mjs +2 -2
- package/src/winenv.mjs +1 -1
- package/src/workspace-api.mjs +21 -4
- package/vendor/client-sdk/index.cjs +5 -4
- package/vendor/cloud-cli/mlai.mjs +184 -28
- package/worker/architecture_worker.py +33 -4
- package/worker/browser_tool.py +19 -1
- package/worker/build_planner.py +75 -7
- package/worker/chat_fork.py +4 -1
- package/worker/checkpoints.py +71 -8
- package/worker/cloud_pty.py +91 -2
- package/worker/command_safety.py +80 -3
- package/worker/compaction_brief.py +186 -0
- package/worker/connected_turn.py +30 -0
- package/worker/context_manager.py +6 -0
- package/worker/conversation.py +139 -12
- package/worker/deck_render.js +11 -0
- package/worker/deck_tools.py +825 -0
- package/worker/git_sync.py +42 -17
- package/worker/goals.py +4 -0
- package/worker/image_generation.py +102 -23
- package/worker/kimi_bridge.py +580 -52
- package/worker/kimi_gates.py +74 -6
- package/worker/kimi_hook.py +213 -19
- package/worker/media_tools.py +182 -0
- package/worker/memory.py +2 -2
- package/worker/mlai_hooks.py +314 -5
- package/worker/mlai_mcp.py +49 -11
- package/worker/mlai_review/README.md +35 -0
- package/worker/mlai_review/__init__.py +1 -0
- package/worker/mlai_review/__main__.py +261 -0
- package/worker/mlai_review/_compat.py +29 -0
- package/worker/mlai_review/analysis/__init__.py +3 -0
- package/worker/mlai_review/analysis/detect.py +50 -0
- package/worker/mlai_review/analysis/main.py +128 -0
- package/worker/mlai_review/analysis/messages.py +8 -0
- package/worker/mlai_review/analysis/tools.py +194 -0
- package/worker/mlai_review/application/__init__.py +1 -0
- package/worker/mlai_review/application/asking.py +53 -0
- package/worker/mlai_review/application/challengers.py +142 -0
- package/worker/mlai_review/application/context_builder.py +194 -0
- package/worker/mlai_review/application/errors.py +17 -0
- package/worker/mlai_review/application/feedback.py +52 -0
- package/worker/mlai_review/application/matching.py +46 -0
- package/worker/mlai_review/application/messages.py +41 -0
- package/worker/mlai_review/application/ports.py +222 -0
- package/worker/mlai_review/application/prompts.py +342 -0
- package/worker/mlai_review/application/review_service.py +200 -0
- package/worker/mlai_review/application/reviewers.py +438 -0
- package/worker/mlai_review/application/streaming.py +91 -0
- package/worker/mlai_review/application/tools.py +138 -0
- package/worker/mlai_review/application/usage.py +145 -0
- package/worker/mlai_review/application/write_check.py +46 -0
- package/worker/mlai_review/contracts/__init__.py +5 -0
- package/worker/mlai_review/contracts/analysis.py +57 -0
- package/worker/mlai_review/contracts/review.py +42 -0
- package/worker/mlai_review/domain/__init__.py +4 -0
- package/worker/mlai_review/domain/changes.py +77 -0
- package/worker/mlai_review/domain/context.py +147 -0
- package/worker/mlai_review/domain/diagnostics.py +239 -0
- package/worker/mlai_review/domain/feedback.py +216 -0
- package/worker/mlai_review/domain/findings.py +505 -0
- package/worker/mlai_review/domain/lifecycle.py +85 -0
- package/worker/mlai_review/domain/partition.py +42 -0
- package/worker/mlai_review/domain/relations.py +166 -0
- package/worker/mlai_review/domain/relevance.py +202 -0
- package/worker/mlai_review/domain/report.py +99 -0
- package/worker/mlai_review/domain/structure.py +56 -0
- package/worker/mlai_review/domain/writes.py +86 -0
- package/worker/mlai_review/eval.py +221 -0
- package/worker/mlai_review/eval_cases/clean-rename.json +11 -0
- package/worker/mlai_review/eval_cases/missing-ownership-check.json +22 -0
- package/worker/mlai_review/eval_cases/off-by-one.json +22 -0
- package/worker/mlai_review/eval_cases/sql-injection.json +20 -0
- package/worker/mlai_review/feedback_report.py +60 -0
- package/worker/mlai_review/infrastructure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/none.py +12 -0
- package/worker/mlai_review/infrastructure/analysis/translate.py +17 -0
- package/worker/mlai_review/infrastructure/github/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/github/client.py +445 -0
- package/worker/mlai_review/infrastructure/github/feedback.py +212 -0
- package/worker/mlai_review/infrastructure/github/http.py +31 -0
- package/worker/mlai_review/infrastructure/github/rendering.py +398 -0
- package/worker/mlai_review/infrastructure/github/token.py +20 -0
- package/worker/mlai_review/infrastructure/local/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/local/analyzer.py +48 -0
- package/worker/mlai_review/infrastructure/local/events.py +100 -0
- package/worker/mlai_review/infrastructure/local/git_reader.py +233 -0
- package/worker/mlai_review/infrastructure/local/publisher.py +138 -0
- package/worker/mlai_review/infrastructure/messages.py +54 -0
- package/worker/mlai_review/infrastructure/models/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/models/openai_compatible.py +269 -0
- package/worker/mlai_review/infrastructure/structure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/structure/heuristic.py +186 -0
- package/worker/mlai_review/infrastructure/structure/tree_sitter.py +186 -0
- package/worker/mlai_review/infrastructure/workspace/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/workspace/local.py +105 -0
- package/worker/mlai_review/infrastructure/workspace/memory.py +77 -0
- package/worker/mlai_review/infrastructure/workspace/paths.py +17 -0
- package/worker/mlai_review/pipeline.py +89 -0
- package/worker/mlai_review/runtime.py +194 -0
- package/worker/mlai_review/settings.py +139 -0
- package/worker/model_stream.py +21 -7
- package/worker/offline_guard.py +1 -1
- package/worker/preview.py +62 -4
- package/worker/product_facts.py +20 -0
- package/worker/project_context.py +33 -7
- package/worker/reply_language.py +92 -10
- package/worker/repo_policy.py +5 -0
- package/worker/roles.json +199 -37
- package/worker/roles.py +271 -9
- package/worker/runner.py +366 -53
- package/worker/runtime_control.py +64 -3
- package/worker/secret_scan.py +3 -2
- package/worker/server_memory.py +122 -0
- package/worker/site_check.py +279 -0
- package/worker/skill_library.py +15 -3
- package/worker/skill_pack.py +170 -16
- package/worker/subagents.py +6 -2
- package/worker/tool_catalog.py +4 -3
- package/worker/verification.py +156 -19
- package/worker/wakeups.py +56 -4
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""A workspace over a directory on this worker's disk."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import fnmatch
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from mlai_review.application.errors import ForbiddenPath
|
|
12
|
+
from mlai_review.application.ports import (
|
|
13
|
+
MAX_GREP_MATCHES,
|
|
14
|
+
MAX_READ_BYTES,
|
|
15
|
+
GrepMatch,
|
|
16
|
+
Workspace,
|
|
17
|
+
)
|
|
18
|
+
from mlai_review.domain.relevance import is_noise
|
|
19
|
+
from mlai_review.infrastructure import messages
|
|
20
|
+
from mlai_review.infrastructure.workspace.paths import relative_path
|
|
21
|
+
|
|
22
|
+
_MAX_MATCH_CHARS = 300
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class LocalWorkspace(Workspace):
|
|
26
|
+
"""Every path is resolved through the real filesystem, so a symlink that points out of the
|
|
27
|
+
checkout is refused the same way `..` is. Searches skip noise files (vendored, generated,
|
|
28
|
+
binaries) and never follow symlinks."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, root: Path) -> None:
|
|
31
|
+
self._root = root.resolve()
|
|
32
|
+
self._listing: list[Path] | None = None
|
|
33
|
+
|
|
34
|
+
async def read(self, path: str, *, max_bytes: int = MAX_READ_BYTES) -> str | None:
|
|
35
|
+
file = self._resolve(path)
|
|
36
|
+
return await asyncio.to_thread(_read_text, file, max_bytes)
|
|
37
|
+
|
|
38
|
+
async def grep(
|
|
39
|
+
self, pattern: str, *, glob: str | None = None, limit: int = MAX_GREP_MATCHES
|
|
40
|
+
) -> tuple[GrepMatch, ...]:
|
|
41
|
+
compiled = re.compile(pattern)
|
|
42
|
+
return await asyncio.to_thread(self._grep, compiled, glob, limit)
|
|
43
|
+
|
|
44
|
+
async def list_dir(self, path: str = "") -> tuple[str, ...]:
|
|
45
|
+
directory = self._resolve(path)
|
|
46
|
+
if not directory.is_dir():
|
|
47
|
+
return ()
|
|
48
|
+
return tuple(
|
|
49
|
+
sorted(
|
|
50
|
+
entry.name + "/" if entry.is_dir() else entry.name
|
|
51
|
+
for entry in directory.iterdir()
|
|
52
|
+
if entry.name != ".git"
|
|
53
|
+
)
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
def _resolve(self, path: str) -> Path:
|
|
57
|
+
resolved = (self._root / relative_path(path)).resolve()
|
|
58
|
+
if resolved != self._root and not resolved.is_relative_to(self._root):
|
|
59
|
+
raise ForbiddenPath(messages.PATH_OUTSIDE_WORKSPACE.format(path=path))
|
|
60
|
+
if ".git" in resolved.relative_to(self._root).parts:
|
|
61
|
+
raise ForbiddenPath(messages.PATH_OUTSIDE_WORKSPACE.format(path=path))
|
|
62
|
+
return resolved
|
|
63
|
+
|
|
64
|
+
def _grep(
|
|
65
|
+
self, compiled: re.Pattern[str], glob: str | None, limit: int
|
|
66
|
+
) -> tuple[GrepMatch, ...]:
|
|
67
|
+
matches: list[GrepMatch] = []
|
|
68
|
+
for file in self._files():
|
|
69
|
+
relative = file.relative_to(self._root).as_posix()
|
|
70
|
+
if glob and not (
|
|
71
|
+
fnmatch.fnmatchcase(relative, glob) or fnmatch.fnmatchcase(relative, f"*/{glob}")
|
|
72
|
+
):
|
|
73
|
+
continue
|
|
74
|
+
text = _read_text(file, MAX_READ_BYTES)
|
|
75
|
+
if text is None:
|
|
76
|
+
continue
|
|
77
|
+
for number, line in enumerate(text.splitlines(), 1):
|
|
78
|
+
if compiled.search(line):
|
|
79
|
+
matches.append(GrepMatch(relative, number, line.rstrip()[:_MAX_MATCH_CHARS]))
|
|
80
|
+
if len(matches) >= limit:
|
|
81
|
+
return tuple(matches)
|
|
82
|
+
return tuple(matches)
|
|
83
|
+
|
|
84
|
+
def _files(self) -> list[Path]:
|
|
85
|
+
"""Regular, non-noise files under the root, walked once per workspace."""
|
|
86
|
+
if self._listing is None:
|
|
87
|
+
found: list[Path] = []
|
|
88
|
+
for directory, subdirectories, names in os.walk(self._root):
|
|
89
|
+
subdirectories[:] = sorted(d for d in subdirectories if d != ".git")
|
|
90
|
+
for name in sorted(names):
|
|
91
|
+
file = Path(directory, name)
|
|
92
|
+
relative = file.relative_to(self._root).as_posix()
|
|
93
|
+
if not file.is_symlink() and file.is_file() and not is_noise(relative):
|
|
94
|
+
found.append(file)
|
|
95
|
+
self._listing = found
|
|
96
|
+
return self._listing
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _read_text(file: Path, max_bytes: int) -> str | None:
|
|
100
|
+
if not file.is_file() or file.stat().st_size > max_bytes:
|
|
101
|
+
return None
|
|
102
|
+
try:
|
|
103
|
+
return file.read_bytes().decode("utf-8")
|
|
104
|
+
except UnicodeDecodeError:
|
|
105
|
+
return None # Binary: nothing to review as text.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""In-memory workspaces for tests and offline evaluation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import fnmatch
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import AsyncIterator, Mapping
|
|
8
|
+
from contextlib import asynccontextmanager
|
|
9
|
+
|
|
10
|
+
from mlai_review.contracts.review import ReviewPullRequestInput
|
|
11
|
+
from mlai_review.application.errors import ReviewError
|
|
12
|
+
from mlai_review.application.ports import (
|
|
13
|
+
MAX_GREP_MATCHES,
|
|
14
|
+
MAX_READ_BYTES,
|
|
15
|
+
GrepMatch,
|
|
16
|
+
Workspace,
|
|
17
|
+
WorkspaceProvider,
|
|
18
|
+
)
|
|
19
|
+
from mlai_review.infrastructure.workspace.paths import relative_path
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class MemoryWorkspace(Workspace):
|
|
23
|
+
def __init__(self, files: Mapping[str, str]) -> None:
|
|
24
|
+
self._files = dict(files)
|
|
25
|
+
|
|
26
|
+
async def read(self, path: str, *, max_bytes: int = MAX_READ_BYTES) -> str | None:
|
|
27
|
+
text = self._files.get(relative_path(path).as_posix())
|
|
28
|
+
if text is None or len(text.encode()) > max_bytes:
|
|
29
|
+
return None
|
|
30
|
+
return text
|
|
31
|
+
|
|
32
|
+
async def grep(
|
|
33
|
+
self, pattern: str, *, glob: str | None = None, limit: int = MAX_GREP_MATCHES
|
|
34
|
+
) -> tuple[GrepMatch, ...]:
|
|
35
|
+
compiled = re.compile(pattern)
|
|
36
|
+
matches: list[GrepMatch] = []
|
|
37
|
+
for path, text in sorted(self._files.items()):
|
|
38
|
+
if glob and not (
|
|
39
|
+
fnmatch.fnmatchcase(path, glob) or fnmatch.fnmatchcase(path, f"*/{glob}")
|
|
40
|
+
):
|
|
41
|
+
continue
|
|
42
|
+
for number, line in enumerate(text.splitlines(), 1):
|
|
43
|
+
if compiled.search(line):
|
|
44
|
+
matches.append(GrepMatch(path, number, line.rstrip()))
|
|
45
|
+
if len(matches) >= limit:
|
|
46
|
+
return tuple(matches)
|
|
47
|
+
return tuple(matches)
|
|
48
|
+
|
|
49
|
+
async def list_dir(self, path: str = "") -> tuple[str, ...]:
|
|
50
|
+
prefix = relative_path(path).as_posix().rstrip("/")
|
|
51
|
+
prefix = f"{prefix}/" if prefix and prefix != "." else ""
|
|
52
|
+
entries: set[str] = set()
|
|
53
|
+
for file in self._files:
|
|
54
|
+
if file.startswith(prefix):
|
|
55
|
+
head, separator, _ = file[len(prefix) :].partition("/")
|
|
56
|
+
entries.add(head + "/" if separator else head)
|
|
57
|
+
return tuple(sorted(entries))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class MemoryWorkspaceProvider(WorkspaceProvider):
|
|
61
|
+
"""Hands out one fixed workspace, or fails the way a real provider would."""
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self, workspace: Workspace | None = None, *, error: ReviewError | None = None
|
|
65
|
+
) -> None:
|
|
66
|
+
self._workspace = workspace or MemoryWorkspace({})
|
|
67
|
+
self._error = error
|
|
68
|
+
self.closed = 0
|
|
69
|
+
|
|
70
|
+
@asynccontextmanager
|
|
71
|
+
async def open(self, target: ReviewPullRequestInput) -> AsyncIterator[Workspace]:
|
|
72
|
+
if self._error is not None:
|
|
73
|
+
raise self._error
|
|
74
|
+
try:
|
|
75
|
+
yield self._workspace
|
|
76
|
+
finally:
|
|
77
|
+
self.closed += 1
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""The one rule every workspace adapter applies to a path before touching anything."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import PurePosixPath
|
|
6
|
+
|
|
7
|
+
from mlai_review.application.errors import ForbiddenPath
|
|
8
|
+
from mlai_review.infrastructure import messages
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def relative_path(path: str) -> PurePosixPath:
|
|
12
|
+
"""A repository-relative path, or ForbiddenPath if it could reach outside the checkout or
|
|
13
|
+
into git's own data."""
|
|
14
|
+
pure = PurePosixPath(path)
|
|
15
|
+
if pure.is_absolute() or "\\" in path or any(p in (".git", "..") for p in pure.parts):
|
|
16
|
+
raise ForbiddenPath(messages.PATH_OUTSIDE_WORKSPACE.format(path=path))
|
|
17
|
+
return pure
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""The order of a review, its timeouts and retries: what the workflow engine did in the
|
|
2
|
+
original service, here in one process. Resolve the head, show the review as started, analyse,
|
|
3
|
+
publish; if analysing fails for good, close the publication so it never spins forever."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import logging
|
|
9
|
+
from collections.abc import Awaitable, Callable
|
|
10
|
+
from typing import TypeVar
|
|
11
|
+
|
|
12
|
+
from mlai_review._compat import TIMEOUTS
|
|
13
|
+
from mlai_review.application.errors import PermanentReviewError, ReviewError
|
|
14
|
+
from mlai_review.application.review_service import ReviewService
|
|
15
|
+
from mlai_review.contracts.review import ReviewPullRequestInput
|
|
16
|
+
from mlai_review.domain.report import ReviewReport
|
|
17
|
+
|
|
18
|
+
T = TypeVar("T")
|
|
19
|
+
_log = logging.getLogger("mlai.review.pipeline")
|
|
20
|
+
|
|
21
|
+
GITHUB_ATTEMPTS = 5
|
|
22
|
+
GITHUB_FIRST_DELAY = 2.0
|
|
23
|
+
# Checkout, static analysis, every reading and refuter call, with margin.
|
|
24
|
+
ANALYSIS_TIMEOUT_SECONDS = 15 * 60.0
|
|
25
|
+
ANALYSIS_ATTEMPTS = 3
|
|
26
|
+
ANALYSIS_DELAY = 5.0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ReviewTimedOut(ReviewError):
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
async def retrying(
|
|
34
|
+
step: Callable[[], Awaitable[T]], *, attempts: int, delay: float, backoff: float = 2.0
|
|
35
|
+
) -> T:
|
|
36
|
+
"""Retry what may pass on retry (ReviewError); a PermanentReviewError fails at once."""
|
|
37
|
+
for attempt in range(1, attempts + 1):
|
|
38
|
+
try:
|
|
39
|
+
return await step()
|
|
40
|
+
except PermanentReviewError:
|
|
41
|
+
raise
|
|
42
|
+
except ReviewError as exc:
|
|
43
|
+
if attempt == attempts:
|
|
44
|
+
raise
|
|
45
|
+
_log.warning("attempt %d failed, retrying: %s", attempt, exc)
|
|
46
|
+
await asyncio.sleep(delay)
|
|
47
|
+
delay *= backoff
|
|
48
|
+
raise AssertionError("unreachable")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
async def review(
|
|
52
|
+
service: ReviewService,
|
|
53
|
+
target: ReviewPullRequestInput,
|
|
54
|
+
*,
|
|
55
|
+
analysis_timeout: float = ANALYSIS_TIMEOUT_SECONDS,
|
|
56
|
+
github_delay: float = GITHUB_FIRST_DELAY,
|
|
57
|
+
analysis_delay: float = ANALYSIS_DELAY,
|
|
58
|
+
) -> ReviewReport:
|
|
59
|
+
target = await retrying(
|
|
60
|
+
lambda: service.resolve(target), attempts=GITHUB_ATTEMPTS, delay=github_delay
|
|
61
|
+
)
|
|
62
|
+
publication = await retrying(
|
|
63
|
+
lambda: service.begin(target), attempts=GITHUB_ATTEMPTS, delay=github_delay
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
async def analyse() -> ReviewReport:
|
|
67
|
+
try:
|
|
68
|
+
return await asyncio.wait_for(service.analyse(target), analysis_timeout)
|
|
69
|
+
except TIMEOUTS as exc:
|
|
70
|
+
raise ReviewTimedOut(f"review took longer than {analysis_timeout:.0f}s") from exc
|
|
71
|
+
|
|
72
|
+
try:
|
|
73
|
+
report = await retrying(analyse, attempts=ANALYSIS_ATTEMPTS, delay=analysis_delay)
|
|
74
|
+
except BaseException:
|
|
75
|
+
try:
|
|
76
|
+
await retrying(
|
|
77
|
+
lambda: service.abandon(target, publication),
|
|
78
|
+
attempts=GITHUB_ATTEMPTS,
|
|
79
|
+
delay=github_delay,
|
|
80
|
+
)
|
|
81
|
+
except Exception as exc: # the original failure is what matters
|
|
82
|
+
_log.warning("could not close the review: %s", exc)
|
|
83
|
+
raise
|
|
84
|
+
await retrying(
|
|
85
|
+
lambda: service.publish(target, publication, report),
|
|
86
|
+
attempts=GITHUB_ATTEMPTS,
|
|
87
|
+
delay=github_delay,
|
|
88
|
+
)
|
|
89
|
+
return report
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Composition root: the only module that chooses concrete implementations. Builds the review
|
|
2
|
+
service for a local checkout (a chat's sandbox or this Mac) or for a pull request on GitHub."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import AsyncIterator
|
|
7
|
+
from contextlib import AsyncExitStack, asynccontextmanager
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import httpx
|
|
12
|
+
|
|
13
|
+
from mlai_review.application.challengers import LlmChallenger
|
|
14
|
+
from mlai_review.application.context_builder import ContextBuilder
|
|
15
|
+
from mlai_review.application.feedback import TeamFeedback
|
|
16
|
+
from mlai_review.application.matching import LlmIssueMatcher
|
|
17
|
+
from mlai_review.application.ports import (
|
|
18
|
+
LanguageModel,
|
|
19
|
+
Outliner,
|
|
20
|
+
PullRequestReader,
|
|
21
|
+
Repository,
|
|
22
|
+
ReviewPublisher,
|
|
23
|
+
StaticAnalyzer,
|
|
24
|
+
Workspace,
|
|
25
|
+
WorkspaceProvider,
|
|
26
|
+
)
|
|
27
|
+
from mlai_review.application.review_service import ReviewService
|
|
28
|
+
from mlai_review.application.reviewers import SPECIALISTS, LlmReviewer
|
|
29
|
+
from mlai_review.application.tools import LookupBudget
|
|
30
|
+
from mlai_review.application.write_check import LlmWriteCheck
|
|
31
|
+
from mlai_review.contracts.review import ReviewPullRequestInput
|
|
32
|
+
from mlai_review.domain.feedback import Dismissal
|
|
33
|
+
from mlai_review.domain.findings import NoiseBudgetPolicy
|
|
34
|
+
from mlai_review.domain.relevance import ReviewLimits
|
|
35
|
+
from mlai_review.infrastructure.github.client import (
|
|
36
|
+
GitHubPullRequestReader,
|
|
37
|
+
GitHubReviewPublisher,
|
|
38
|
+
)
|
|
39
|
+
from mlai_review.infrastructure.github.feedback import GitHubFeedbackSource
|
|
40
|
+
from mlai_review.infrastructure.github.token import StaticTokenProvider
|
|
41
|
+
from mlai_review.infrastructure.models.openai_compatible import OpenAICompatibleModel
|
|
42
|
+
from mlai_review.infrastructure.structure.tree_sitter import TreeSitterOutliner
|
|
43
|
+
from mlai_review.infrastructure.workspace.local import LocalWorkspace
|
|
44
|
+
from mlai_review.settings import ModelSettings, ReviewSettings
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class CheckoutProvider(WorkspaceProvider):
|
|
48
|
+
"""The checkout is already on disk (the chat's working tree, or the clone the server made
|
|
49
|
+
for a pull request); the review reads it and never deletes it."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, root: Path) -> None:
|
|
52
|
+
self._root = root
|
|
53
|
+
|
|
54
|
+
@asynccontextmanager
|
|
55
|
+
async def open(self, target: ReviewPullRequestInput) -> AsyncIterator[Workspace]:
|
|
56
|
+
yield LocalWorkspace(self._root)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class StaticDismissals:
|
|
60
|
+
"""Dismissals the app recorded ("Not useful" on a finding), in TeamFeedback's shape."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, dismissed: tuple[Dismissal, ...]) -> None:
|
|
63
|
+
self._dismissed = dismissed
|
|
64
|
+
|
|
65
|
+
async def dismissed(self, repository: Repository) -> tuple[Dismissal, ...]:
|
|
66
|
+
return self._dismissed
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True, slots=True)
|
|
70
|
+
class Built:
|
|
71
|
+
service: ReviewService
|
|
72
|
+
resources: AsyncExitStack
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
async def model_for(
|
|
76
|
+
settings: ModelSettings, api_key: str, resources: AsyncExitStack, review: ReviewSettings
|
|
77
|
+
) -> OpenAICompatibleModel:
|
|
78
|
+
http = await resources.enter_async_context(
|
|
79
|
+
httpx.AsyncClient(
|
|
80
|
+
base_url=settings.endpoint.rstrip("/") + "/", timeout=settings.timeout_seconds
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
return OpenAICompatibleModel(
|
|
84
|
+
http=http,
|
|
85
|
+
api_key=api_key,
|
|
86
|
+
model=settings.model,
|
|
87
|
+
reasoning_effort=settings.effort(review.reasoning_effort),
|
|
88
|
+
max_tokens=settings.max_tokens,
|
|
89
|
+
extra_body=settings.extra_body(),
|
|
90
|
+
json_mode=settings.json_mode,
|
|
91
|
+
max_concurrent=settings.max_concurrent,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def reviewer_for(
|
|
96
|
+
model: LanguageModel, settings: ModelSettings, review: ReviewSettings
|
|
97
|
+
) -> LlmReviewer:
|
|
98
|
+
lookups = (
|
|
99
|
+
LookupBudget(
|
|
100
|
+
max_calls=review.lookups_per_specialist, max_result_chars=review.lookup_result_chars
|
|
101
|
+
)
|
|
102
|
+
if review.lookups_per_specialist > 0
|
|
103
|
+
else None
|
|
104
|
+
)
|
|
105
|
+
refuter_lookups = (
|
|
106
|
+
LookupBudget(max_calls=review.refuter_lookups, max_result_chars=review.lookup_result_chars)
|
|
107
|
+
if review.refuter_lookups > 0
|
|
108
|
+
else None
|
|
109
|
+
)
|
|
110
|
+
thinking = getattr(model, "thinking", None) # a fake or replayed model thinks as it is
|
|
111
|
+
refuting: LanguageModel = (
|
|
112
|
+
thinking(
|
|
113
|
+
settings.effort(review.refuter_effort),
|
|
114
|
+
review.refuter_max_tokens or settings.max_tokens,
|
|
115
|
+
)
|
|
116
|
+
if review.refuter_effort and thinking is not None
|
|
117
|
+
else model
|
|
118
|
+
)
|
|
119
|
+
return LlmReviewer(
|
|
120
|
+
model,
|
|
121
|
+
NoiseBudgetPolicy(**review.policy),
|
|
122
|
+
tuple(SPECIALISTS[name] for name in review.specialists),
|
|
123
|
+
lookups,
|
|
124
|
+
LlmChallenger(refuting, refuter_lookups) if review.refuter else None,
|
|
125
|
+
passes=review.passes,
|
|
126
|
+
matcher=LlmIssueMatcher(model) if review.issue_matching else None,
|
|
127
|
+
part_chars=review.part_chars,
|
|
128
|
+
write_check=LlmWriteCheck(refuting) if review.write_check else None,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
async def build_local(
|
|
133
|
+
*,
|
|
134
|
+
root: Path,
|
|
135
|
+
reader: PullRequestReader,
|
|
136
|
+
publisher: ReviewPublisher,
|
|
137
|
+
model: ModelSettings,
|
|
138
|
+
api_key: str,
|
|
139
|
+
review: ReviewSettings,
|
|
140
|
+
analyzer: StaticAnalyzer | None = None,
|
|
141
|
+
dismissed: tuple[Dismissal, ...] = (),
|
|
142
|
+
outliner: Outliner | None = None,
|
|
143
|
+
) -> Built:
|
|
144
|
+
resources = AsyncExitStack()
|
|
145
|
+
llm = await model_for(model, api_key, resources, review)
|
|
146
|
+
service = ReviewService(
|
|
147
|
+
context=ContextBuilder(reader, outliner or TreeSitterOutliner()),
|
|
148
|
+
workspaces=CheckoutProvider(root),
|
|
149
|
+
publisher=publisher,
|
|
150
|
+
reviewer=reviewer_for(llm, model, review),
|
|
151
|
+
limits=ReviewLimits(max_files=review.max_files, max_diff_lines=review.max_diff_lines),
|
|
152
|
+
analyzer=analyzer,
|
|
153
|
+
analysis_wait_seconds=review.analysis_wait_seconds,
|
|
154
|
+
feedback=StaticDismissals(dismissed) if review.team_feedback else None, # type: ignore[arg-type]
|
|
155
|
+
ignore=review.ignore,
|
|
156
|
+
min_severity=review.min_severity,
|
|
157
|
+
instructions=review.instructions,
|
|
158
|
+
)
|
|
159
|
+
return Built(service, resources)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
async def build_pull_request(
|
|
163
|
+
*,
|
|
164
|
+
root: Path,
|
|
165
|
+
model: ModelSettings,
|
|
166
|
+
api_key: str,
|
|
167
|
+
github_token: str,
|
|
168
|
+
review: ReviewSettings,
|
|
169
|
+
analyzer: StaticAnalyzer | None = None,
|
|
170
|
+
api_url: str = "https://api.github.com",
|
|
171
|
+
details_url: str | None = None,
|
|
172
|
+
) -> Built:
|
|
173
|
+
resources = AsyncExitStack()
|
|
174
|
+
github = await resources.enter_async_context(httpx.AsyncClient(base_url=api_url, timeout=30.0))
|
|
175
|
+
tokens = StaticTokenProvider(github_token)
|
|
176
|
+
llm = await model_for(model, api_key, resources, review)
|
|
177
|
+
service = ReviewService(
|
|
178
|
+
context=ContextBuilder(GitHubPullRequestReader(http=github, tokens=tokens), TreeSitterOutliner()), # type: ignore[arg-type]
|
|
179
|
+
workspaces=CheckoutProvider(root),
|
|
180
|
+
publisher=GitHubReviewPublisher(http=github, tokens=tokens, details_url=details_url), # type: ignore[arg-type]
|
|
181
|
+
reviewer=reviewer_for(llm, model, review),
|
|
182
|
+
limits=ReviewLimits(max_files=review.max_files, max_diff_lines=review.max_diff_lines),
|
|
183
|
+
analyzer=analyzer,
|
|
184
|
+
analysis_wait_seconds=review.analysis_wait_seconds,
|
|
185
|
+
feedback=(
|
|
186
|
+
TeamFeedback(GitHubFeedbackSource(http=github, tokens=tokens)) # type: ignore[arg-type]
|
|
187
|
+
if review.team_feedback
|
|
188
|
+
else None
|
|
189
|
+
),
|
|
190
|
+
ignore=review.ignore,
|
|
191
|
+
min_severity=review.min_severity,
|
|
192
|
+
instructions=review.instructions,
|
|
193
|
+
)
|
|
194
|
+
return Built(service, resources)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""How a review is run, per mode. Replaces the service's environment settings: the app picks a
|
|
2
|
+
mode and a model, everything else follows from the mode. The numbers are the reviewer's
|
|
3
|
+
measured defaults (see GUIDE.md in the original service); deep and security change only what
|
|
4
|
+
those modes are for."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field, replace
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
MODES = ("standard", "deep", "security")
|
|
12
|
+
# Providers whose endpoint accepts reasoning_effort, and the values it accepts.
|
|
13
|
+
_EFFORTS = {"isoquant": ("low", "high", "max")}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True, slots=True)
|
|
17
|
+
class ModelSettings:
|
|
18
|
+
endpoint: str
|
|
19
|
+
model: str
|
|
20
|
+
provider: str = ""
|
|
21
|
+
max_tokens: int = 4_000 # reasoning models think first; below ~1500 replies come back empty
|
|
22
|
+
timeout_seconds: float = 180.0
|
|
23
|
+
max_concurrent: int = 6
|
|
24
|
+
json_mode: bool = True
|
|
25
|
+
|
|
26
|
+
def effort(self, wanted: str | None) -> str | None:
|
|
27
|
+
"""The effort to send, or None where the endpoint takes none."""
|
|
28
|
+
accepted = _EFFORTS.get(self.provider)
|
|
29
|
+
if not wanted or accepted is None:
|
|
30
|
+
return None
|
|
31
|
+
return wanted if wanted in accepted else accepted[0]
|
|
32
|
+
|
|
33
|
+
def extra_body(self) -> dict[str, Any]:
|
|
34
|
+
return {"include_reasoning": False} if self.provider == "isoquant" else {}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True, slots=True)
|
|
38
|
+
class ReviewSettings:
|
|
39
|
+
mode: str = "standard"
|
|
40
|
+
specialists: tuple[str, ...] = ("general",)
|
|
41
|
+
passes: int = 3
|
|
42
|
+
issue_matching: bool = True
|
|
43
|
+
part_chars: int = 0
|
|
44
|
+
reasoning_effort: str | None = "low"
|
|
45
|
+
lookups_per_specialist: int = 0
|
|
46
|
+
lookup_result_chars: int = 40_000
|
|
47
|
+
refuter: bool = True
|
|
48
|
+
refuter_effort: str | None = None
|
|
49
|
+
refuter_max_tokens: int | None = None
|
|
50
|
+
refuter_lookups: int = 0
|
|
51
|
+
write_check: bool = False
|
|
52
|
+
team_feedback: bool = True
|
|
53
|
+
max_files: int = 300
|
|
54
|
+
max_diff_lines: int = 20_000
|
|
55
|
+
analysis_wait_seconds: float = 240.0
|
|
56
|
+
# From the repository's settings (the app, overridden by .mlai/review.yml): changed paths
|
|
57
|
+
# the reviewer skips, the lowest severity commented inline, and the team's own guidance.
|
|
58
|
+
ignore: tuple[str, ...] = ()
|
|
59
|
+
min_severity: str = "low"
|
|
60
|
+
instructions: str = ""
|
|
61
|
+
policy: dict[str, Any] = field(
|
|
62
|
+
default_factory=lambda: {
|
|
63
|
+
"publish_at": 0.90,
|
|
64
|
+
"publish_high_at": 0.75,
|
|
65
|
+
"publish_confirmed_at": 0.75,
|
|
66
|
+
"publish_tentative_at": 0.50,
|
|
67
|
+
"tentative_budget": 3,
|
|
68
|
+
"note_budget": 8,
|
|
69
|
+
"publish_agreed_at": 2,
|
|
70
|
+
"publish_single_confirmed_at": 0.80,
|
|
71
|
+
}
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def for_mode(mode: str, *, scope: str = "changes") -> ReviewSettings:
|
|
76
|
+
"""standard: three independent readings, grouped and matched, each issue challenged once.
|
|
77
|
+
deep: five readings; the refuter thinks hard and may look around the checkout (the
|
|
78
|
+
adversarial pass); the write check tests every data-changing write. security: the security
|
|
79
|
+
specialist alone, its claims challenged as hard as in deep mode. A repository scan is
|
|
80
|
+
reviewed in parts of related files, with a larger budget for what is shown."""
|
|
81
|
+
base = ReviewSettings()
|
|
82
|
+
if mode == "deep":
|
|
83
|
+
base = replace(
|
|
84
|
+
base,
|
|
85
|
+
mode="deep",
|
|
86
|
+
passes=5,
|
|
87
|
+
refuter_effort="high",
|
|
88
|
+
refuter_max_tokens=24_000,
|
|
89
|
+
refuter_lookups=6,
|
|
90
|
+
write_check=True,
|
|
91
|
+
)
|
|
92
|
+
elif mode == "security":
|
|
93
|
+
base = replace(
|
|
94
|
+
base,
|
|
95
|
+
mode="security",
|
|
96
|
+
specialists=("security",),
|
|
97
|
+
refuter_effort="high",
|
|
98
|
+
refuter_max_tokens=24_000,
|
|
99
|
+
refuter_lookups=4,
|
|
100
|
+
)
|
|
101
|
+
if scope == "repo":
|
|
102
|
+
policy = {**base.policy, "budget": 12, "note_budget": 20}
|
|
103
|
+
base = replace(base, part_chars=60_000, policy=policy, team_feedback=False)
|
|
104
|
+
return base
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
SEVERITIES = ("low", "medium", "high", "critical")
|
|
108
|
+
MAX_IGNORE = 30
|
|
109
|
+
MAX_INSTRUCTIONS = 2_000
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def with_repository(base: ReviewSettings, raw: object) -> ReviewSettings:
|
|
113
|
+
"""The mode's settings with the repository's own: ignored paths, minimum severity,
|
|
114
|
+
guidance and the size cap. Anything malformed is left at the mode's value."""
|
|
115
|
+
if not isinstance(raw, dict):
|
|
116
|
+
return base
|
|
117
|
+
ignore = tuple(
|
|
118
|
+
str(g).strip()
|
|
119
|
+
for g in (raw.get("ignore") or ())
|
|
120
|
+
if isinstance(g, str) and g.strip() and len(g) <= 200 and ".." not in g
|
|
121
|
+
)[:MAX_IGNORE]
|
|
122
|
+
severity = raw.get("minSeverity")
|
|
123
|
+
instructions = raw.get("instructions")
|
|
124
|
+
|
|
125
|
+
def cap(value: object, default: int, ceiling: int) -> int:
|
|
126
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 1:
|
|
127
|
+
return default
|
|
128
|
+
return min(int(value), ceiling)
|
|
129
|
+
|
|
130
|
+
return replace(
|
|
131
|
+
base,
|
|
132
|
+
ignore=ignore,
|
|
133
|
+
min_severity=severity if severity in SEVERITIES else base.min_severity,
|
|
134
|
+
instructions=instructions.strip()[:MAX_INSTRUCTIONS]
|
|
135
|
+
if isinstance(instructions, str)
|
|
136
|
+
else base.instructions,
|
|
137
|
+
max_files=cap(raw.get("maxFiles"), base.max_files, 3_000),
|
|
138
|
+
max_diff_lines=cap(raw.get("maxLines"), base.max_diff_lines, 100_000),
|
|
139
|
+
)
|
package/worker/model_stream.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
"""OpenAI-compatible SSE assembly. Partial tool arguments are never executable."""
|
|
2
2
|
import json,queue,threading,time,urllib.request
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
THOUGHT_MAX=12000
|
|
5
|
+
|
|
6
|
+
def read_stream(response,on_text,on_thought=None):
|
|
7
|
+
content=[];thought=[];calls={};usage={};finished=False;total=0;reason=None
|
|
6
8
|
for raw in response:
|
|
7
9
|
total+=len(raw)
|
|
8
10
|
if total>4000000 or len(raw)>500000:raise ValueError('Model stream exceeded size limit')
|
|
@@ -19,7 +21,14 @@ def read_stream(response,on_text):
|
|
|
19
21
|
delta=choice.get('delta',{})
|
|
20
22
|
text=delta.get('content')
|
|
21
23
|
if isinstance(text,str):content.append(text);on_text(text)
|
|
22
|
-
#
|
|
24
|
+
# Reasoning never joins the content or tool calls (it is not executed or sent back to the model); when asked
|
|
25
|
+
# for, it is collected separately so the activity view can show the model's thought process.
|
|
26
|
+
for key in ('reasoning_content','reasoning'):
|
|
27
|
+
piece=delta.get(key)
|
|
28
|
+
if isinstance(piece,str) and piece:
|
|
29
|
+
thought.append(piece)
|
|
30
|
+
if on_thought:on_thought(piece)
|
|
31
|
+
break
|
|
23
32
|
for item in delta.get('tool_calls',[]):
|
|
24
33
|
index=item.get('index',0)
|
|
25
34
|
if type(index)!=int or not 0<=index<8:raise ValueError('Invalid streamed tool index')
|
|
@@ -29,9 +38,11 @@ def read_stream(response,on_text):
|
|
|
29
38
|
fragment=item.get('function',{}).get(key)
|
|
30
39
|
if isinstance(fragment,str):call['function'][key]+=fragment
|
|
31
40
|
if not finished or reason not in ('stop','tool_calls'):raise ValueError('Incomplete model stream; no actions were executed')
|
|
32
|
-
|
|
41
|
+
result={'choices':[{'message':{'content':''.join(content),'tool_calls':[calls[i] for i in sorted(calls)]},'finish_reason':reason}],'usage':usage}
|
|
42
|
+
if thought:result['thought_display']=''.join(thought)[-THOUGHT_MAX:]
|
|
43
|
+
return result
|
|
33
44
|
|
|
34
|
-
def request_stream(request,cancelled,on_progress,timeout=240):
|
|
45
|
+
def request_stream(request,cancelled,on_progress,timeout=240,on_thought=None):
|
|
35
46
|
events=queue.Queue(maxsize=128);stop=threading.Event();response_holder=[]
|
|
36
47
|
def send(kind,value):
|
|
37
48
|
while not stop.is_set():
|
|
@@ -46,11 +57,11 @@ def request_stream(request,cancelled,on_progress,timeout=240):
|
|
|
46
57
|
raw=response.read(4000001)
|
|
47
58
|
if len(raw)>4000000:raise ValueError('Model response exceeded size limit')
|
|
48
59
|
result=json.loads(raw)
|
|
49
|
-
else:result=read_stream(response,lambda text:send('text',text))
|
|
60
|
+
else:result=read_stream(response,lambda text:send('text',text),(lambda text:send('thought',text)) if on_thought else None)
|
|
50
61
|
send('result',result)
|
|
51
62
|
except Exception as exc:send('error',exc)
|
|
52
63
|
threading.Thread(target=fetch,daemon=True).start()
|
|
53
|
-
start=time.monotonic();last=0;parts=[];first=None
|
|
64
|
+
start=time.monotonic();last=0;parts=[];first=None;thought=[];thought_last=0
|
|
54
65
|
try:
|
|
55
66
|
while True:
|
|
56
67
|
cancelled();now=time.monotonic()
|
|
@@ -64,6 +75,9 @@ def request_stream(request,cancelled,on_progress,timeout=240):
|
|
|
64
75
|
if first is None:first=time.monotonic()-start
|
|
65
76
|
parts.append(value)
|
|
66
77
|
if now-last>=.5:on_progress(''.join(parts),now-start,first);last=now
|
|
78
|
+
elif kind=='thought':
|
|
79
|
+
thought.append(value)
|
|
80
|
+
if on_thought and now-thought_last>=.5:on_thought(''.join(thought)[-THOUGHT_MAX:],now-start);thought_last=now
|
|
67
81
|
elif kind=='error':raise value
|
|
68
82
|
else:
|
|
69
83
|
on_progress(''.join(parts),time.monotonic()-start,first)
|