mlai-code 0.3.3 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +1 -1
  2. package/package.json +1 -1
  3. package/src/checkpoints.mjs +217 -0
  4. package/src/cli.mjs +153 -26
  5. package/src/cloud.mjs +2 -0
  6. package/src/engine.mjs +1 -1
  7. package/src/headless.mjs +39 -11
  8. package/src/home.mjs +3 -2
  9. package/src/host.mjs +141 -13
  10. package/src/ide-quick.mjs +147 -0
  11. package/src/ide.mjs +182 -0
  12. package/src/netbox.mjs +1 -1
  13. package/src/parity.mjs +17 -0
  14. package/src/relay.mjs +1 -1
  15. package/src/render.mjs +16 -5
  16. package/src/runtime.mjs +2 -0
  17. package/src/server.mjs +46 -7
  18. package/src/session.mjs +31 -8
  19. package/src/tui/acp.mjs +121 -19
  20. package/src/tui/app.mjs +2 -1
  21. package/src/tui/chat.mjs +35 -10
  22. package/src/tui/cmd-config.mjs +3 -1
  23. package/src/tui/cmd-core.mjs +9 -5
  24. package/src/tui/cmd-extensibility.mjs +5 -0
  25. package/src/tui/extensions-local.mjs +2 -0
  26. package/src/tui/mcp-config.mjs +22 -4
  27. package/src/tui/review-cli.mjs +15 -4
  28. package/src/tui/snapshots.mjs +57 -16
  29. package/src/tui/transcript.mjs +13 -2
  30. package/src/two-words.mjs +14 -8
  31. package/src/view-sync.mjs +48 -0
  32. package/src/winenv.mjs +1 -1
  33. package/src/workspace-api.mjs +21 -4
  34. package/vendor/client-sdk/index.cjs +2 -1
  35. package/vendor/cloud-cli/mlai.mjs +23 -11
  36. package/worker/architecture_worker.py +33 -4
  37. package/worker/browser_tool.py +19 -1
  38. package/worker/chat_fork.py +4 -1
  39. package/worker/checkpoints.py +71 -8
  40. package/worker/command_safety.py +56 -3
  41. package/worker/connected_turn.py +30 -0
  42. package/worker/conversation.py +125 -13
  43. package/worker/git_sync.py +4 -4
  44. package/worker/image_generation.py +2 -2
  45. package/worker/kimi_bridge.py +251 -31
  46. package/worker/kimi_gates.py +25 -6
  47. package/worker/kimi_hook.py +154 -13
  48. package/worker/media_tools.py +4 -1
  49. package/worker/mlai_hooks.py +222 -3
  50. package/worker/mlai_mcp.py +26 -6
  51. package/worker/mlai_review/README.md +35 -0
  52. package/worker/mlai_review/__init__.py +1 -0
  53. package/worker/mlai_review/__main__.py +261 -0
  54. package/worker/mlai_review/_compat.py +29 -0
  55. package/worker/mlai_review/analysis/__init__.py +3 -0
  56. package/worker/mlai_review/analysis/detect.py +50 -0
  57. package/worker/mlai_review/analysis/main.py +128 -0
  58. package/worker/mlai_review/analysis/messages.py +8 -0
  59. package/worker/mlai_review/analysis/tools.py +194 -0
  60. package/worker/mlai_review/application/__init__.py +1 -0
  61. package/worker/mlai_review/application/asking.py +53 -0
  62. package/worker/mlai_review/application/challengers.py +142 -0
  63. package/worker/mlai_review/application/context_builder.py +194 -0
  64. package/worker/mlai_review/application/errors.py +17 -0
  65. package/worker/mlai_review/application/feedback.py +52 -0
  66. package/worker/mlai_review/application/matching.py +46 -0
  67. package/worker/mlai_review/application/messages.py +41 -0
  68. package/worker/mlai_review/application/ports.py +222 -0
  69. package/worker/mlai_review/application/prompts.py +342 -0
  70. package/worker/mlai_review/application/review_service.py +200 -0
  71. package/worker/mlai_review/application/reviewers.py +438 -0
  72. package/worker/mlai_review/application/streaming.py +91 -0
  73. package/worker/mlai_review/application/tools.py +138 -0
  74. package/worker/mlai_review/application/usage.py +145 -0
  75. package/worker/mlai_review/application/write_check.py +46 -0
  76. package/worker/mlai_review/contracts/__init__.py +5 -0
  77. package/worker/mlai_review/contracts/analysis.py +57 -0
  78. package/worker/mlai_review/contracts/review.py +42 -0
  79. package/worker/mlai_review/domain/__init__.py +4 -0
  80. package/worker/mlai_review/domain/changes.py +77 -0
  81. package/worker/mlai_review/domain/context.py +147 -0
  82. package/worker/mlai_review/domain/diagnostics.py +239 -0
  83. package/worker/mlai_review/domain/feedback.py +216 -0
  84. package/worker/mlai_review/domain/findings.py +505 -0
  85. package/worker/mlai_review/domain/lifecycle.py +85 -0
  86. package/worker/mlai_review/domain/partition.py +42 -0
  87. package/worker/mlai_review/domain/relations.py +166 -0
  88. package/worker/mlai_review/domain/relevance.py +202 -0
  89. package/worker/mlai_review/domain/report.py +99 -0
  90. package/worker/mlai_review/domain/structure.py +56 -0
  91. package/worker/mlai_review/domain/writes.py +86 -0
  92. package/worker/mlai_review/eval.py +221 -0
  93. package/worker/mlai_review/eval_cases/clean-rename.json +11 -0
  94. package/worker/mlai_review/eval_cases/missing-ownership-check.json +22 -0
  95. package/worker/mlai_review/eval_cases/off-by-one.json +22 -0
  96. package/worker/mlai_review/eval_cases/sql-injection.json +20 -0
  97. package/worker/mlai_review/feedback_report.py +60 -0
  98. package/worker/mlai_review/infrastructure/__init__.py +1 -0
  99. package/worker/mlai_review/infrastructure/analysis/__init__.py +1 -0
  100. package/worker/mlai_review/infrastructure/analysis/none.py +12 -0
  101. package/worker/mlai_review/infrastructure/analysis/translate.py +17 -0
  102. package/worker/mlai_review/infrastructure/github/__init__.py +0 -0
  103. package/worker/mlai_review/infrastructure/github/client.py +445 -0
  104. package/worker/mlai_review/infrastructure/github/feedback.py +212 -0
  105. package/worker/mlai_review/infrastructure/github/http.py +31 -0
  106. package/worker/mlai_review/infrastructure/github/rendering.py +398 -0
  107. package/worker/mlai_review/infrastructure/github/token.py +20 -0
  108. package/worker/mlai_review/infrastructure/local/__init__.py +1 -0
  109. package/worker/mlai_review/infrastructure/local/analyzer.py +48 -0
  110. package/worker/mlai_review/infrastructure/local/events.py +100 -0
  111. package/worker/mlai_review/infrastructure/local/git_reader.py +233 -0
  112. package/worker/mlai_review/infrastructure/local/publisher.py +138 -0
  113. package/worker/mlai_review/infrastructure/messages.py +54 -0
  114. package/worker/mlai_review/infrastructure/models/__init__.py +0 -0
  115. package/worker/mlai_review/infrastructure/models/openai_compatible.py +269 -0
  116. package/worker/mlai_review/infrastructure/structure/__init__.py +1 -0
  117. package/worker/mlai_review/infrastructure/structure/heuristic.py +186 -0
  118. package/worker/mlai_review/infrastructure/structure/tree_sitter.py +186 -0
  119. package/worker/mlai_review/infrastructure/workspace/__init__.py +1 -0
  120. package/worker/mlai_review/infrastructure/workspace/local.py +105 -0
  121. package/worker/mlai_review/infrastructure/workspace/memory.py +77 -0
  122. package/worker/mlai_review/infrastructure/workspace/paths.py +17 -0
  123. package/worker/mlai_review/pipeline.py +89 -0
  124. package/worker/mlai_review/runtime.py +194 -0
  125. package/worker/mlai_review/settings.py +139 -0
  126. package/worker/preview.py +15 -0
  127. package/worker/product_facts.py +3 -1
  128. package/worker/repo_policy.py +5 -0
  129. package/worker/roles.py +29 -2
  130. package/worker/runner.py +164 -23
  131. package/worker/runtime_control.py +95 -9
  132. package/worker/secret_scan.py +3 -2
  133. package/worker/server_memory.py +122 -0
  134. package/worker/subagents.py +6 -2
  135. package/worker/tool_catalog.py +5 -2
  136. package/worker/verification.py +113 -8
@@ -0,0 +1,261 @@
1
+ """ML AI Review, run from the command line.
2
+
3
+ python -m mlai_review run REQUEST.json review a checkout or a pull request
4
+ python -m mlai_review analyse --root DIR --out FILE [--no-eslint] [--no-install]
5
+ python -m mlai_review feedback owner/name [--days 14] (GITHUB_TOKEN)
6
+ python -m mlai_review eval [--cases DIR] [--mode standard] [--record]
7
+
8
+ The request names the checkout, the scope, the mode and the model endpoint; the model key is
9
+ read from MODEL_API_KEY and a GitHub token from GITHUB_TOKEN, never from the request. Progress
10
+ and findings are appended to the request's `out` file as JSON lines (see events.py).
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import asyncio
17
+ import json
18
+ import logging
19
+ import os
20
+ import sys
21
+ import uuid
22
+ from datetime import datetime, timedelta, timezone
23
+ from pathlib import Path
24
+ from typing import Any
25
+
26
+ from mlai_review import pipeline
27
+ from mlai_review.application.errors import AccessDenied
28
+ from mlai_review.application.usage import CreditsExhausted, metering
29
+ from mlai_review.contracts.review import ReviewPullRequestInput
30
+ from mlai_review.infrastructure.local.analyzer import LocalAnalyzer, StoredAnalysis
31
+ from mlai_review.infrastructure.local.events import EventLog, report_json
32
+ from mlai_review.infrastructure.local.git_reader import LocalGitReader
33
+ from mlai_review.infrastructure.local.publisher import EventPublisher, read_dismissals
34
+ from mlai_review.runtime import build_local, build_pull_request
35
+ from mlai_review.settings import MODES, ModelSettings, for_mode, with_repository
36
+
37
+ FAILED = "The review could not finish. Nothing was changed; try again."
38
+ GITHUB_DENIED = (
39
+ "GitHub refused the token in your Vault. Give it Contents and Pull requests (read and write)"
40
+ " access to this repository, then review again."
41
+ )
42
+ CREDITS_OUT = (
43
+ "You have used all your credits for this month, so the review stopped early. Findings so far"
44
+ " are kept. Pick a plan or buy a top-up in Settings > Billing, then review again."
45
+ )
46
+
47
+
48
+ def budget_of(value: object) -> int | None:
49
+ """Billable tokens the review may use (the account's remaining credits); None is no cap."""
50
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
51
+ return None
52
+ return max(0, int(value))
53
+
54
+
55
+ def model_settings(raw: dict[str, Any]) -> ModelSettings:
56
+ endpoint = str(raw.get("endpoint") or "").strip()
57
+ name = str(raw.get("model") or "").strip()
58
+ if not endpoint or not name:
59
+ raise SystemExit("request.model needs endpoint and model")
60
+ return ModelSettings(
61
+ endpoint=endpoint,
62
+ model=name,
63
+ provider=str(raw.get("provider") or ""),
64
+ json_mode=raw.get("jsonMode", True) is not False,
65
+ max_concurrent=max(1, min(12, int(raw.get("maxConcurrent") or 6))),
66
+ )
67
+
68
+
69
+ async def run(request: dict[str, Any]) -> int:
70
+ log = EventLog(Path(request["out"]) if request.get("out") else None)
71
+ review_id = str(request.get("id") or uuid.uuid4().hex[:12])
72
+ mode = request.get("mode") if request.get("mode") in MODES else "standard"
73
+ scope = str(request.get("scope") or "changes")
74
+ root = Path(str(request.get("root") or ".")).resolve()
75
+ api_key = os.environ.pop("MODEL_API_KEY", "")
76
+ log.emit("started", id=review_id, mode=mode, scope=scope, pid=os.getpid())
77
+ if not api_key:
78
+ log.emit("error", message="The model is not configured for reviews.")
79
+ return 2
80
+ model = model_settings(request.get("model") or {})
81
+ settings = with_repository(for_mode(mode, scope=scope), request.get("settings"))
82
+ analysis = request.get("analysis") or "auto"
83
+ analyzer = (
84
+ None
85
+ if analysis == "off"
86
+ else StoredAnalysis(Path(analysis))
87
+ if analysis not in ("auto", "local")
88
+ else LocalAnalyzer(root)
89
+ )
90
+ pr = request.get("pullRequest")
91
+ with metering(rates=request.get("creditRates"), budget=budget_of(request.get("budget"))) as used:
92
+ try:
93
+ if pr:
94
+ github_token = os.environ.pop("GITHUB_TOKEN", "")
95
+ if not github_token:
96
+ log.emit("error", message="GitHub is not connected for this repository.")
97
+ return 2
98
+ built = await build_pull_request(
99
+ root=root,
100
+ model=model,
101
+ api_key=api_key,
102
+ github_token=github_token,
103
+ review=settings,
104
+ analyzer=analyzer if analysis not in ("auto", "local") else None,
105
+ details_url=pr.get("detailsUrl"),
106
+ api_url=os.environ.get("MLAI_REVIEW_GITHUB_API") or "https://api.github.com",
107
+ )
108
+ target = ReviewPullRequestInput(
109
+ installation_id=0,
110
+ repository_id=0,
111
+ repository_full_name=str(pr["repository"]),
112
+ pull_number=int(pr["number"]),
113
+ head_sha=str(pr.get("headSha") or ""),
114
+ base_sha=str(pr.get("baseSha") or ""),
115
+ triggered_by_delivery=str(pr.get("delivery") or review_id),
116
+ previous_head_sha=pr.get("previousHeadSha") or None,
117
+ )
118
+ async with built.resources:
119
+ report = await pipeline.review(built.service, target)
120
+ _limited(log, used)
121
+ log.emit("done", report=report_json(report), usage=_usage(used))
122
+ return 0
123
+ reader = LocalGitReader(
124
+ root,
125
+ scope=scope,
126
+ base=str(request.get("base") or ""),
127
+ head=str(request.get("head") or ""),
128
+ title=str(request.get("title") or ""),
129
+ body=str(request.get("body") or ""),
130
+ paths=tuple(str(p) for p in request.get("paths") or ()),
131
+ )
132
+ out = Path(str(request.get("out") or ""))
133
+ publisher = EventPublisher(log, root, prior=_prior_logs(request.get("prior"), out))
134
+ dismissed = read_dismissals(
135
+ Path(request["dismissed"]) if request.get("dismissed") else None
136
+ )
137
+ built = await build_local(
138
+ root=root,
139
+ reader=reader,
140
+ publisher=publisher,
141
+ model=model,
142
+ api_key=api_key,
143
+ review=settings,
144
+ analyzer=analyzer,
145
+ dismissed=dismissed,
146
+ )
147
+ head, base = await reader.head(_local_target(root, "", ""))
148
+ async with built.resources:
149
+ await pipeline.review(
150
+ built.service, _local_target(root, head or "0" * 40, base or head or "0" * 40)
151
+ )
152
+ _limited(log, used)
153
+ log.emit("usage", usage=_usage(used))
154
+ return 0
155
+ except Exception as exc: # reported to the app; details stay in the log on stderr
156
+ if isinstance(exc, CreditsExhausted) or used.exhausted:
157
+ log.emit("error", message=CREDITS_OUT, detail="credits", limited=True, usage=_usage(used))
158
+ return 3
159
+ logging.getLogger("mlai.review").exception("review failed")
160
+ message = GITHUB_DENIED if pr and isinstance(exc, AccessDenied) else FAILED
161
+ log.emit("error", message=message, detail=type(exc).__name__, usage=_usage(used))
162
+ return 1
163
+
164
+
165
+ def _limited(log: EventLog, used: Any) -> None:
166
+ """Readings that stopped at the credit cap still finish the review; say why it is partial."""
167
+ if used.exhausted:
168
+ log.emit("limited", message=CREDITS_OUT, usage=_usage(used))
169
+
170
+
171
+ def _local_target(root: Path, head: str, base: str) -> ReviewPullRequestInput:
172
+ return ReviewPullRequestInput(
173
+ installation_id=0,
174
+ repository_id=0,
175
+ repository_full_name=f"local/{root.name or 'workspace'}",
176
+ pull_number=0,
177
+ head_sha=head,
178
+ base_sha=base,
179
+ triggered_by_delivery="local",
180
+ )
181
+
182
+
183
+ def _prior_logs(value: object, out: Path) -> tuple[Path, ...]:
184
+ """Earlier reviews' event logs, oldest first. A folder stands for every log in it."""
185
+ files: list[Path] = []
186
+ for item in value if isinstance(value, list) else [value] if value else []:
187
+ path = Path(str(item))
188
+ if path.is_dir():
189
+ files.extend(sorted(path.glob("*.jsonl"), key=lambda f: f.stat().st_mtime))
190
+ else:
191
+ files.append(path)
192
+ return tuple(f for f in files if f.resolve() != out.resolve())
193
+
194
+
195
+ def _usage(used: Any) -> dict[str, int]:
196
+ return {
197
+ "calls": used.calls,
198
+ "promptTokens": used.prompt_tokens,
199
+ "cachedTokens": used.cached_tokens,
200
+ "completionTokens": used.completion_tokens,
201
+ "billable": used.billable,
202
+ }
203
+
204
+
205
+ async def feedback(repositories: list[str], days: int) -> None:
206
+ import httpx
207
+
208
+ from mlai_review.application.feedback import FeedbackReporter
209
+ from mlai_review.feedback_report import render
210
+ from mlai_review.infrastructure.github.feedback import GitHubFeedbackSource
211
+ from mlai_review.infrastructure.github.token import StaticTokenProvider
212
+
213
+ token = os.environ.pop("GITHUB_TOKEN", "")
214
+ since = datetime.now(timezone.utc) - timedelta(days=days)
215
+ async with httpx.AsyncClient(base_url="https://api.github.com", timeout=30.0) as http:
216
+ reporter = FeedbackReporter(GitHubFeedbackSource(http=http, tokens=StaticTokenProvider(token))) # type: ignore[arg-type]
217
+ for full_name in repositories:
218
+ summary = await reporter.report(full_name, since)
219
+ sys.stdout.write(render(full_name, days, summary) + "\n")
220
+
221
+
222
+ def main(argv: list[str] | None = None) -> int:
223
+ logging.basicConfig(level=logging.INFO, stream=sys.stderr, format="%(name)s: %(message)s")
224
+ parser = argparse.ArgumentParser(prog="mlai_review", description=__doc__)
225
+ sub = parser.add_subparsers(dest="command", required=True)
226
+ run_parser = sub.add_parser("run")
227
+ run_parser.add_argument("request", type=Path)
228
+ analyse_parser = sub.add_parser("analyse")
229
+ analyse_parser.add_argument("--root", type=Path, required=True)
230
+ analyse_parser.add_argument("--out", type=Path, required=True)
231
+ analyse_parser.add_argument("--no-eslint", action="store_true")
232
+ analyse_parser.add_argument("--no-install", action="store_true", help="never install packages")
233
+ feedback_parser = sub.add_parser("feedback")
234
+ feedback_parser.add_argument("repositories", nargs="+")
235
+ feedback_parser.add_argument("--days", type=int, default=14)
236
+ eval_parser = sub.add_parser("eval")
237
+ eval_parser.add_argument("--cases", type=Path, default=None)
238
+ eval_parser.add_argument("--mode", default=None, choices=MODES)
239
+ eval_parser.add_argument("--record", action="store_true", help="call the model, save replies")
240
+ eval_parser.add_argument("--request", type=Path, help="model endpoint, as in run")
241
+ args = parser.parse_args(argv)
242
+ if args.command == "run":
243
+ return asyncio.run(run(json.loads(args.request.read_text(encoding="utf-8"))))
244
+ if args.command == "analyse":
245
+ from mlai_review.analysis.detect import Tool
246
+ from mlai_review.analysis.main import analyse
247
+
248
+ allowed = tuple(t for t in Tool if t is not Tool.ESLINT) if args.no_eslint else None
249
+ result = analyse(args.root, allowed=allowed, install=not args.no_install)
250
+ args.out.write_text(json.dumps(result.as_dict()), encoding="utf-8")
251
+ return 0
252
+ if args.command == "feedback":
253
+ asyncio.run(feedback(args.repositories, args.days))
254
+ return 0
255
+ from mlai_review.eval import main as eval_main
256
+
257
+ return eval_main(args)
258
+
259
+
260
+ if __name__ == "__main__":
261
+ sys.exit(main())
@@ -0,0 +1,29 @@
1
+ """Python 3.10 support: the engine on a Mac may run 3.10, the reviewer was written for 3.13."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ from enum import Enum
7
+ from typing import Any
8
+
9
+ try:
10
+ from enum import StrEnum
11
+ except ImportError: # Python 3.10
12
+
13
+ class StrEnum(str, Enum): # type: ignore[no-redef]
14
+ def __str__(self) -> str:
15
+ return str(self.value)
16
+
17
+ def __format__(self, spec: str) -> str:
18
+ return format(str(self.value), spec)
19
+
20
+
21
+ try:
22
+ from typing import Self
23
+ except ImportError: # Python 3.10
24
+ Self = Any # type: ignore[assignment,misc]
25
+
26
+ # asyncio.wait_for raises asyncio.TimeoutError, which is the builtin only from 3.11.
27
+ TIMEOUTS = (TimeoutError, asyncio.TimeoutError)
28
+
29
+ __all__ = ["TIMEOUTS", "Self", "StrEnum"]
@@ -0,0 +1,3 @@
1
+ """The static-analysis sandbox: clone one commit, run the tools the repository calls for,
2
+ report their messages. Runs as a job with no secrets beyond a short-lived, single-repository
3
+ token and nowhere to write except its result."""
@@ -0,0 +1,50 @@
1
+ """Which tools a repository calls for, from the files it contains. Nothing here is specific to
2
+ a customer: a tsconfig means TypeScript, a pyproject means Python, and so on."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from mlai_review._compat import StrEnum
7
+ from pathlib import Path
8
+
9
+
10
+ class Tool(StrEnum):
11
+ RUFF = "ruff"
12
+ TSC = "tsc"
13
+ ESLINT = "eslint"
14
+
15
+
16
+ _ESLINT_CONFIGS = (
17
+ "eslint.config.js",
18
+ "eslint.config.mjs",
19
+ "eslint.config.cjs",
20
+ "eslint.config.ts",
21
+ ".eslintrc",
22
+ ".eslintrc.js",
23
+ ".eslintrc.cjs",
24
+ ".eslintrc.json",
25
+ ".eslintrc.yml",
26
+ ".eslintrc.yaml",
27
+ )
28
+ _MAX_SCAN = 20_000 # files to look at when deciding whether Python is present
29
+
30
+
31
+ def detect(root: Path) -> tuple[Tool, ...]:
32
+ tools: list[Tool] = []
33
+ if _has_python(root):
34
+ tools.append(Tool.RUFF)
35
+ if (root / "tsconfig.json").is_file():
36
+ tools.append(Tool.TSC)
37
+ if any((root / name).is_file() for name in _ESLINT_CONFIGS):
38
+ tools.append(Tool.ESLINT)
39
+ return tuple(tools)
40
+
41
+
42
+ def _has_python(root: Path) -> bool:
43
+ if any((root / name).is_file() for name in ("pyproject.toml", "setup.py", "setup.cfg")):
44
+ return True
45
+ for i, path in enumerate(root.rglob("*.py")):
46
+ if ".git" not in path.parts and "node_modules" not in path.parts:
47
+ return True
48
+ if i > _MAX_SCAN:
49
+ break
50
+ return False
@@ -0,0 +1,128 @@
1
+ """Entry point of the sandbox job.
2
+
3
+ Environment:
4
+ MLAI_REPOSITORY owner/name
5
+ MLAI_HEAD_SHA the commit to analyse
6
+ MLAI_GITHUB_TOKEN a short-lived installation token with contents:read on that one repository
7
+ MLAI_RESULT_URL where to PUT the result (a write-only blob URL); omitted: print to stdout
8
+ MLAI_CLONE_BASE defaults to https://github.com
9
+
10
+ Local use: `mlai-analyse --root /path/to/checkout` skips the clone and prints the result.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import base64
17
+ import json
18
+ import os
19
+ import shutil
20
+ import subprocess
21
+ import sys
22
+ import tempfile
23
+ import urllib.request
24
+ from pathlib import Path
25
+
26
+ from mlai_review.analysis.detect import Tool, detect
27
+ from mlai_review.analysis.tools import ToolFailed, install_node_dependencies, run_tool
28
+ from mlai_review.contracts.analysis import AnalysisResult, Diagnostic
29
+
30
+ CLONE_TIMEOUT_SECONDS = 600
31
+
32
+
33
+ def analyse(
34
+ root: Path, *, allowed: tuple[Tool, ...] | None = None, install: bool = True
35
+ ) -> AnalysisResult:
36
+ """`allowed` limits the tools (eslint runs the repository's own config, which is code);
37
+ `install` False uses the dependencies already there, never touching a developer's
38
+ node_modules."""
39
+ tools = tuple(t for t in detect(root) if allowed is None or t in allowed)
40
+ diagnostics: list[Diagnostic] = []
41
+ errors: dict[str, str] = {}
42
+ ran: list[str] = []
43
+ node = Tool.TSC in tools or Tool.ESLINT in tools
44
+ if install and node and not (root / "node_modules").is_dir():
45
+ try:
46
+ install_node_dependencies(root)
47
+ except ToolFailed as exc:
48
+ errors["npm"] = str(exc)
49
+ for tool in tools:
50
+ try:
51
+ diagnostics += run_tool(tool, root)
52
+ ran.append(tool.value)
53
+ except ToolFailed as exc:
54
+ errors[tool.value] = str(exc)
55
+ return AnalysisResult(tools_run=tuple(ran), diagnostics=tuple(diagnostics), errors=errors)
56
+
57
+
58
+ def clone(repository: str, sha: str, token: str, base: str, into: Path) -> None:
59
+ credentials = base64.b64encode(f"x-access-token:{token}".encode()).decode()
60
+ env = {
61
+ **os.environ,
62
+ "GIT_TERMINAL_PROMPT": "0",
63
+ "GIT_CONFIG_COUNT": "1",
64
+ "GIT_CONFIG_KEY_0": "http.extraheader",
65
+ "GIT_CONFIG_VALUE_0": f"Authorization: Basic {credentials}",
66
+ }
67
+ url = f"{base.rstrip('/')}/{repository}.git"
68
+ for args in (
69
+ ["init", "-q"],
70
+ ["fetch", "-q", "--depth", "1", "--no-tags", url, sha],
71
+ ["checkout", "-q", "--detach", "FETCH_HEAD"],
72
+ ):
73
+ subprocess.run(
74
+ ["git", "-C", str(into), *args], check=True, env=env, timeout=CLONE_TIMEOUT_SECONDS
75
+ )
76
+
77
+
78
+ def publish(result: AnalysisResult, url: str | None) -> None:
79
+ body = json.dumps(result.as_dict()).encode()
80
+ if not url:
81
+ sys.stdout.write(body.decode() + "\n")
82
+ return
83
+ request = urllib.request.Request(
84
+ url,
85
+ data=body,
86
+ method="PUT",
87
+ headers={"x-ms-blob-type": "BlockBlob", "Content-Type": "application/json"},
88
+ )
89
+ with urllib.request.urlopen(request, timeout=60):
90
+ pass
91
+
92
+
93
+ def main() -> int:
94
+ parser = argparse.ArgumentParser(description=__doc__)
95
+ parser.add_argument("--root", type=Path, help="analyse an existing checkout instead of cloning")
96
+ parser.add_argument("--out", type=Path, help="write the result here instead of stdout")
97
+ parser.add_argument("--no-eslint", action="store_true", help="skip eslint (runs repo code)")
98
+ args = parser.parse_args()
99
+ if args.root:
100
+ allowed = tuple(t for t in Tool if t is not Tool.ESLINT) if args.no_eslint else None
101
+ result = analyse(args.root, allowed=allowed)
102
+ if args.out:
103
+ args.out.write_text(json.dumps(result.as_dict()))
104
+ else:
105
+ publish(result, None)
106
+ return 0
107
+ repository = os.environ["MLAI_REPOSITORY"]
108
+ sha = os.environ["MLAI_HEAD_SHA"]
109
+ token = os.environ.pop("MLAI_GITHUB_TOKEN") # out of the environment before any tool runs
110
+ result_url = os.environ.get("MLAI_RESULT_URL")
111
+ base = os.environ.get("MLAI_CLONE_BASE", "https://github.com")
112
+ workdir = Path(tempfile.mkdtemp(prefix="mlai-analysis-"))
113
+ root = workdir / "repo"
114
+ root.mkdir()
115
+ try:
116
+ clone(repository, sha, token, base, root)
117
+ del token
118
+ result = analyse(root)
119
+ except subprocess.CalledProcessError as exc:
120
+ result = AnalysisResult(errors={"clone": f"git exited with {exc.returncode}"})
121
+ finally:
122
+ shutil.rmtree(workdir, ignore_errors=True)
123
+ publish(result, result_url)
124
+ return 0
125
+
126
+
127
+ if __name__ == "__main__":
128
+ sys.exit(main())
@@ -0,0 +1,8 @@
1
+ """Wording for why a tool could not run. Reported to operators in the result, never to the
2
+ model."""
3
+
4
+ TOOL_NOT_AVAILABLE = "{name}: not available: {error}"
5
+ TOOL_TIMED_OUT = "{name}: took longer than {seconds:.0f}s"
6
+ TOOL_EXITED = "{name}: exited with {code}: {detail}"
7
+ TOOL_UNREADABLE_OUTPUT = "{name}: unreadable output: {error}"
8
+ ESLINT_MISSING = "eslint: not installed in the repository (node_modules/.bin/eslint)"
@@ -0,0 +1,194 @@
1
+ """Running each tool and turning what it prints into diagnostics. Every tool is run with a
2
+ timeout and a clean environment; its exit code is not an error (tools exit non-zero when they
3
+ find something), only its output matters."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ import os
9
+ import re
10
+ import subprocess
11
+ from collections.abc import Callable
12
+ from pathlib import Path
13
+
14
+ from mlai_review.analysis import messages
15
+ from mlai_review.analysis.detect import Tool
16
+ from mlai_review.contracts.analysis import Diagnostic
17
+
18
+ TOOL_TIMEOUT_SECONDS = 600
19
+ INSTALL_TIMEOUT_SECONDS = 900
20
+ MAX_DIAGNOSTICS = 2_000
21
+
22
+ _TSC_LINE = re.compile(
23
+ r"^(?P<path>[^(]+)\((?P<line>\d+),\d+\): "
24
+ r"(?P<level>error|warning) (?P<rule>TS\d+): (?P<message>.*)$"
25
+ )
26
+ # Ruff rules that mean the code is wrong, not merely untidy: undefined and misused names,
27
+ # syntax errors, invalid constructs. Everything else is reported as a warning.
28
+ _RUFF_ERRORS = frozenset(
29
+ {
30
+ "F821",
31
+ "F822",
32
+ "F823",
33
+ "F811",
34
+ "F632",
35
+ "F601",
36
+ "F602",
37
+ "F631",
38
+ "F633",
39
+ "F701",
40
+ "F702",
41
+ "F704",
42
+ "F706",
43
+ "F707",
44
+ }
45
+ )
46
+
47
+
48
+ class ToolFailed(Exception):
49
+ """The tool could not run at all (missing, crashed, timed out)."""
50
+
51
+
52
+ def run_tool(tool: Tool, root: Path) -> list[Diagnostic]:
53
+ runner: Callable[[Path], list[Diagnostic]] = {
54
+ Tool.RUFF: _ruff,
55
+ Tool.TSC: _tsc,
56
+ Tool.ESLINT: _eslint,
57
+ }[tool]
58
+ return runner(root)[:MAX_DIAGNOSTICS]
59
+
60
+
61
+ def install_node_dependencies(root: Path) -> None:
62
+ """Dependencies for tsc and eslint. Lifecycle scripts never run: they are arbitrary code
63
+ from the pull request, and the tools do not need them."""
64
+ if not (root / "package.json").is_file():
65
+ return
66
+ lockfile = (root / "package-lock.json").is_file() or (root / "npm-shrinkwrap.json").is_file()
67
+ command = [
68
+ "npm",
69
+ "ci" if lockfile else "install",
70
+ "--ignore-scripts",
71
+ "--no-audit",
72
+ "--no-fund",
73
+ ]
74
+ _execute(command, root, INSTALL_TIMEOUT_SECONDS, name="npm")
75
+
76
+
77
+ def _ruff(root: Path) -> list[Diagnostic]:
78
+ output = _execute(
79
+ ["ruff", "check", "--output-format", "json", "--exit-zero", "."],
80
+ root,
81
+ TOOL_TIMEOUT_SECONDS,
82
+ name="ruff",
83
+ )
84
+ try:
85
+ items = json.loads(output or "[]")
86
+ except json.JSONDecodeError as exc:
87
+ raise ToolFailed(messages.TOOL_UNREADABLE_OUTPUT.format(name="ruff", error=exc)) from exc
88
+ return [
89
+ Diagnostic(
90
+ tool="ruff",
91
+ path=_relative(root, item["filename"]),
92
+ line=int(item["location"]["row"]),
93
+ rule=str(item.get("code") or ""),
94
+ message=str(item.get("message", "")),
95
+ level="error" if _is_ruff_error(str(item.get("code") or "")) else "warning",
96
+ )
97
+ for item in items
98
+ ]
99
+
100
+
101
+ def _tsc(root: Path) -> list[Diagnostic]:
102
+ binary = root / "node_modules" / ".bin" / "tsc"
103
+ command = [str(binary) if binary.is_file() else "tsc", "--noEmit", "--pretty", "false"]
104
+ output = _execute(command, root, TOOL_TIMEOUT_SECONDS, name="tsc", ok_codes=(0, 1, 2))
105
+ diagnostics = []
106
+ for line in output.splitlines():
107
+ match = _TSC_LINE.match(line.strip())
108
+ if match:
109
+ diagnostics.append(
110
+ Diagnostic(
111
+ tool="tsc",
112
+ path=_relative(root, match["path"]),
113
+ line=int(match["line"]),
114
+ rule=match["rule"],
115
+ message=match["message"],
116
+ level=match["level"],
117
+ )
118
+ )
119
+ return diagnostics
120
+
121
+
122
+ def _eslint(root: Path) -> list[Diagnostic]:
123
+ binary = root / "node_modules" / ".bin" / "eslint"
124
+ if not binary.is_file():
125
+ raise ToolFailed(messages.ESLINT_MISSING)
126
+ output = _execute(
127
+ [str(binary), ".", "--format", "json", "--no-error-on-unmatched-pattern"],
128
+ root,
129
+ TOOL_TIMEOUT_SECONDS,
130
+ name="eslint",
131
+ ok_codes=(0, 1),
132
+ )
133
+ try:
134
+ files = json.loads(output or "[]")
135
+ except json.JSONDecodeError as exc:
136
+ raise ToolFailed(messages.TOOL_UNREADABLE_OUTPUT.format(name="eslint", error=exc)) from exc
137
+ return [
138
+ Diagnostic(
139
+ tool="eslint",
140
+ path=_relative(root, file["filePath"]),
141
+ line=int(message.get("line") or 1),
142
+ rule=str(message.get("ruleId") or ""),
143
+ message=str(message.get("message", "")),
144
+ level="error" if message.get("severity") == 2 else "warning",
145
+ )
146
+ for file in files
147
+ for message in file.get("messages", [])
148
+ ]
149
+
150
+
151
+ def _execute(
152
+ command: list[str],
153
+ root: Path,
154
+ timeout: float,
155
+ *,
156
+ name: str,
157
+ ok_codes: tuple[int, ...] = (0,),
158
+ ) -> str:
159
+ env = {
160
+ "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
161
+ "HOME": str(root.parent),
162
+ "LANG": "C.UTF-8",
163
+ "CI": "1",
164
+ }
165
+ try:
166
+ done = subprocess.run(
167
+ command, cwd=root, env=env, capture_output=True, text=True, timeout=timeout
168
+ )
169
+ except FileNotFoundError as exc:
170
+ raise ToolFailed(messages.TOOL_NOT_AVAILABLE.format(name=name, error=exc)) from exc
171
+ except subprocess.TimeoutExpired as exc:
172
+ raise ToolFailed(messages.TOOL_TIMED_OUT.format(name=name, seconds=timeout)) from exc
173
+ if done.returncode not in ok_codes:
174
+ detail = (done.stderr or done.stdout).strip().splitlines()
175
+ raise ToolFailed(
176
+ messages.TOOL_EXITED.format(
177
+ name=name, code=done.returncode, detail=detail[-1] if detail else ""
178
+ )
179
+ )
180
+ return done.stdout
181
+
182
+
183
+ def _is_ruff_error(code: str) -> bool:
184
+ return code in _RUFF_ERRORS or code.startswith("E9")
185
+
186
+
187
+ def _relative(root: Path, path: str) -> str:
188
+ candidate = Path(path)
189
+ if candidate.is_absolute():
190
+ try:
191
+ return candidate.resolve().relative_to(root.resolve()).as_posix()
192
+ except ValueError:
193
+ return candidate.as_posix()
194
+ return candidate.as_posix()
@@ -0,0 +1 @@
1
+ """Use cases and the ports they need. Depends on the domain and mlai_contracts only."""