@kernel-sig/console 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/.env.example +50 -0
  2. package/LICENSE +127 -0
  3. package/Makefile +66 -0
  4. package/README.md +279 -0
  5. package/backend/.dockerignore +12 -0
  6. package/backend/Dockerfile +43 -0
  7. package/backend/alembic/env.py +75 -0
  8. package/backend/alembic/script.py.mako +25 -0
  9. package/backend/alembic/versions/0001_initial.py +88 -0
  10. package/backend/alembic/versions/350e2d9b6553_add_pr_comment_table.py +48 -0
  11. package/backend/alembic/versions/4f8342e727fe_sig_info_roster_fields_and_branch_.py +39 -0
  12. package/backend/alembic/versions/5a31ade90136_add_repository_credential_pull_request_.py +293 -0
  13. package/backend/alembic/versions/87a008142f17_add_classification_attention_and_ai_.py +436 -0
  14. package/backend/alembic/versions/a1c7d90e4b52_branch_belongs_to_a_repository.py +72 -0
  15. package/backend/alembic/versions/b41c9d7e5f28_extend_pr_kind_categories.py +43 -0
  16. package/backend/alembic/versions/c8e4f2a71b93_add_release_kind.py +40 -0
  17. package/backend/alembic/versions/d1275091dfdc_add_needs_detail_flag_to_pull_request.py +44 -0
  18. package/backend/alembic/versions/e5b27c9d3a41_downgrade_cve_without_ids.py +54 -0
  19. package/backend/alembic/versions/ee3159a4eff0_add_sig_meeting_member_and_release_.py +164 -0
  20. package/backend/alembic/versions/fcf3c186d63b_attention_rule_subscriptions.py +42 -0
  21. package/backend/alembic.ini +40 -0
  22. package/backend/app/__init__.py +0 -0
  23. package/backend/app/api/__init__.py +0 -0
  24. package/backend/app/api/deps.py +74 -0
  25. package/backend/app/api/v1/__init__.py +0 -0
  26. package/backend/app/api/v1/ai.py +386 -0
  27. package/backend/app/api/v1/analytics.py +48 -0
  28. package/backend/app/api/v1/attention.py +142 -0
  29. package/backend/app/api/v1/auth.py +106 -0
  30. package/backend/app/api/v1/classification.py +192 -0
  31. package/backend/app/api/v1/health.py +29 -0
  32. package/backend/app/api/v1/issues.py +143 -0
  33. package/backend/app/api/v1/pulls.py +388 -0
  34. package/backend/app/api/v1/repositories.py +273 -0
  35. package/backend/app/api/v1/router.py +32 -0
  36. package/backend/app/api/v1/sig.py +565 -0
  37. package/backend/app/api/v1/users.py +87 -0
  38. package/backend/app/api/v1/webhooks.py +135 -0
  39. package/backend/app/core/__init__.py +0 -0
  40. package/backend/app/core/config.py +72 -0
  41. package/backend/app/core/crypto.py +74 -0
  42. package/backend/app/core/db.py +79 -0
  43. package/backend/app/core/exceptions.py +60 -0
  44. package/backend/app/core/logging.py +69 -0
  45. package/backend/app/core/permissions.py +87 -0
  46. package/backend/app/core/queue.py +57 -0
  47. package/backend/app/core/security.py +69 -0
  48. package/backend/app/domain/__init__.py +0 -0
  49. package/backend/app/domain/attention.py +618 -0
  50. package/backend/app/domain/classification.py +689 -0
  51. package/backend/app/domain/meeting.py +408 -0
  52. package/backend/app/domain/release.py +163 -0
  53. package/backend/app/domain/review.py +416 -0
  54. package/backend/app/domain/sig.py +178 -0
  55. package/backend/app/domain/sig_info.py +201 -0
  56. package/backend/app/integrations/__init__.py +0 -0
  57. package/backend/app/integrations/atomgit/__init__.py +0 -0
  58. package/backend/app/integrations/atomgit/client.py +505 -0
  59. package/backend/app/integrations/atomgit/models.py +311 -0
  60. package/backend/app/integrations/llm/__init__.py +0 -0
  61. package/backend/app/integrations/llm/prompts.py +222 -0
  62. package/backend/app/integrations/llm/provider.py +326 -0
  63. package/backend/app/main.py +242 -0
  64. package/backend/app/middleware/__init__.py +0 -0
  65. package/backend/app/middleware/audit.py +129 -0
  66. package/backend/app/middleware/request_context.py +42 -0
  67. package/backend/app/models/__init__.py +112 -0
  68. package/backend/app/models/ai.py +214 -0
  69. package/backend/app/models/attention.py +131 -0
  70. package/backend/app/models/attention_settings.py +61 -0
  71. package/backend/app/models/audit.py +53 -0
  72. package/backend/app/models/base.py +35 -0
  73. package/backend/app/models/classification.py +144 -0
  74. package/backend/app/models/credential.py +56 -0
  75. package/backend/app/models/issue.py +132 -0
  76. package/backend/app/models/meeting.py +189 -0
  77. package/backend/app/models/pull_request.py +288 -0
  78. package/backend/app/models/repository.py +196 -0
  79. package/backend/app/models/sig.py +194 -0
  80. package/backend/app/models/user.py +44 -0
  81. package/backend/app/schemas/__init__.py +0 -0
  82. package/backend/app/schemas/ai.py +120 -0
  83. package/backend/app/schemas/attention.py +43 -0
  84. package/backend/app/schemas/auth.py +26 -0
  85. package/backend/app/schemas/classification.py +58 -0
  86. package/backend/app/schemas/common.py +43 -0
  87. package/backend/app/schemas/pull_request.py +214 -0
  88. package/backend/app/schemas/repository.py +94 -0
  89. package/backend/app/schemas/sig.py +246 -0
  90. package/backend/app/schemas/user.py +79 -0
  91. package/backend/app/services/__init__.py +0 -0
  92. package/backend/app/services/ai_service.py +569 -0
  93. package/backend/app/services/analytics_service.py +390 -0
  94. package/backend/app/services/attention_queue.py +451 -0
  95. package/backend/app/services/attention_service.py +432 -0
  96. package/backend/app/services/auth_service.py +74 -0
  97. package/backend/app/services/classification_service.py +658 -0
  98. package/backend/app/services/credential_service.py +105 -0
  99. package/backend/app/services/pull_query.py +273 -0
  100. package/backend/app/services/release_service.py +446 -0
  101. package/backend/app/services/repository_service.py +132 -0
  102. package/backend/app/services/sig_service.py +385 -0
  103. package/backend/app/services/sync_service.py +752 -0
  104. package/backend/app/services/user_service.py +68 -0
  105. package/backend/app/worker.py +389 -0
  106. package/backend/entrypoint.sh +10 -0
  107. package/backend/pyproject.toml +68 -0
  108. package/backend/tests/test_analysis_api.py +154 -0
  109. package/backend/tests/test_atomgit_client.py +360 -0
  110. package/backend/tests/test_atomgit_models.py +257 -0
  111. package/backend/tests/test_attention.py +291 -0
  112. package/backend/tests/test_audit_middleware.py +135 -0
  113. package/backend/tests/test_classification.py +498 -0
  114. package/backend/tests/test_config.py +41 -0
  115. package/backend/tests/test_crypto.py +68 -0
  116. package/backend/tests/test_exceptions.py +62 -0
  117. package/backend/tests/test_health.py +63 -0
  118. package/backend/tests/test_llm_provider.py +320 -0
  119. package/backend/tests/test_meeting_domain.py +169 -0
  120. package/backend/tests/test_permissions.py +69 -0
  121. package/backend/tests/test_pull_query_wiring.py +66 -0
  122. package/backend/tests/test_pull_schemas.py +82 -0
  123. package/backend/tests/test_release_domain.py +82 -0
  124. package/backend/tests/test_review_parser.py +301 -0
  125. package/backend/tests/test_schemas_user.py +97 -0
  126. package/backend/tests/test_security.py +92 -0
  127. package/cli/index.js +338 -0
  128. package/compose.yaml +105 -0
  129. package/frontend/.dockerignore +4 -0
  130. package/frontend/Dockerfile +27 -0
  131. package/frontend/index.html +14 -0
  132. package/frontend/nginx.conf +47 -0
  133. package/frontend/package-lock.json +5020 -0
  134. package/frontend/package.json +35 -0
  135. package/frontend/src/api/ai.ts +124 -0
  136. package/frontend/src/api/analytics.ts +66 -0
  137. package/frontend/src/api/attention.ts +181 -0
  138. package/frontend/src/api/auth.ts +74 -0
  139. package/frontend/src/api/classification.ts +170 -0
  140. package/frontend/src/api/issues.ts +90 -0
  141. package/frontend/src/api/pulls.ts +233 -0
  142. package/frontend/src/api/repositories.ts +95 -0
  143. package/frontend/src/api/sig.ts +232 -0
  144. package/frontend/src/app/antd-theme.ts +94 -0
  145. package/frontend/src/app/providers.tsx +59 -0
  146. package/frontend/src/app/router.tsx +411 -0
  147. package/frontend/src/app/search.ts +30 -0
  148. package/frontend/src/components/ClassificationBadge.tsx +58 -0
  149. package/frontend/src/components/GateBadge.tsx +13 -0
  150. package/frontend/src/components/SeverityBadge.tsx +20 -0
  151. package/frontend/src/components/layout/AppShell.tsx +16 -0
  152. package/frontend/src/components/layout/AuthLayout.tsx +32 -0
  153. package/frontend/src/components/layout/Sidebar.tsx +223 -0
  154. package/frontend/src/components/layout/TopBar.tsx +47 -0
  155. package/frontend/src/components/pulls/DiscussionTimeline.tsx +145 -0
  156. package/frontend/src/components/pulls/FacetRail.tsx +199 -0
  157. package/frontend/src/components/pulls/LabelChips.tsx +87 -0
  158. package/frontend/src/components/ui/alert.tsx +30 -0
  159. package/frontend/src/components/ui/badge.tsx +53 -0
  160. package/frontend/src/components/ui/button.tsx +51 -0
  161. package/frontend/src/components/ui/card.tsx +64 -0
  162. package/frontend/src/components/ui/chart-theme.ts +65 -0
  163. package/frontend/src/components/ui/data-table.tsx +39 -0
  164. package/frontend/src/components/ui/echart.tsx +70 -0
  165. package/frontend/src/components/ui/empty-state.tsx +22 -0
  166. package/frontend/src/components/ui/input.tsx +39 -0
  167. package/frontend/src/components/ui/lazy-chart.tsx +21 -0
  168. package/frontend/src/components/ui/skeleton.tsx +19 -0
  169. package/frontend/src/hooks/use-current-repository.ts +44 -0
  170. package/frontend/src/hooks/use-current-user.ts +38 -0
  171. package/frontend/src/lib/api-client.ts +93 -0
  172. package/frontend/src/lib/css-color.ts +60 -0
  173. package/frontend/src/lib/utils.ts +45 -0
  174. package/frontend/src/main.tsx +22 -0
  175. package/frontend/src/pages/attention/AttentionQueuePage.tsx +316 -0
  176. package/frontend/src/pages/attention/RuleSettingsPanel.tsx +177 -0
  177. package/frontend/src/pages/branches/BranchDetailPage.tsx +632 -0
  178. package/frontend/src/pages/branches/BranchListPage.tsx +308 -0
  179. package/frontend/src/pages/dashboard/DashboardPage.tsx +657 -0
  180. package/frontend/src/pages/issues/IssueListPage.tsx +284 -0
  181. package/frontend/src/pages/login/LoginPage.tsx +95 -0
  182. package/frontend/src/pages/meetings/MeetingDetailPage.tsx +403 -0
  183. package/frontend/src/pages/meetings/MeetingListPage.tsx +264 -0
  184. package/frontend/src/pages/members/MembersPage.tsx +534 -0
  185. package/frontend/src/pages/pulls/PullDetailPage.tsx +833 -0
  186. package/frontend/src/pages/pulls/PullListPage.tsx +399 -0
  187. package/frontend/src/pages/settings/AiSettingsPage.tsx +449 -0
  188. package/frontend/src/pages/settings/SettingsPage.tsx +321 -0
  189. package/frontend/src/pages/setup/SetupPage.tsx +144 -0
  190. package/frontend/src/styles/globals.css +147 -0
  191. package/frontend/tsconfig.json +22 -0
  192. package/frontend/vite.config.ts +57 -0
  193. package/package.json +48 -0
  194. package/scripts/e2e-auth-flow.py +131 -0
  195. package/scripts/e2e-pr-detail.py +128 -0
  196. package/scripts/e2e-verify.py +773 -0
  197. package/scripts/verify-ai-pipeline.py +616 -0
  198. package/scripts/verify-analysis-pipeline.py +303 -0
@@ -0,0 +1,752 @@
1
+ """数据同步服务。
2
+
3
+ 把 AtomGit 的 PR / Issue 拉取到本地。
4
+
5
+ 同步策略(分层,控制 API 调用量):
6
+ - L1 列表:拉取 PR/Issue 列表,更新标题、状态、标签等轻量字段
7
+ - L2 详情:仅对内容发生变化的 PR 拉取提交与文件
8
+ - L3 评论:仅在评论数增加时拉取,用于解析评审事件
9
+
10
+ 实测 open PR 逾 1100 个,每次全量拉详情需 2000+ 请求,
11
+ 会迅速耗尽上游配额,因此必须按变化驱动而非每次都拉全量。
12
+
13
+ 幂等性:所有写入以 (repository_id, number) 为冲突键做 upsert,
14
+ 并以 AtomGit 的 updated_at 做乐观并发控制 —— 仅当远端更新时才覆盖本地。
15
+ """
16
+
17
+ import hashlib
18
+ import uuid
19
+ from datetime import UTC, datetime, timedelta
20
+
21
+ import structlog
22
+ from sqlalchemy import func, select
23
+ from sqlalchemy.dialects.postgresql import insert as pg_insert
24
+ from sqlalchemy.exc import SQLAlchemyError
25
+ from sqlalchemy.ext.asyncio import AsyncSession
26
+
27
+ from app.core.config import Settings
28
+ from app.core.exceptions import ValidationError
29
+ from app.domain.review import derive_gate, parse_events
30
+ from app.integrations.atomgit.client import AtomGitClient, AtomGitError
31
+ from app.integrations.atomgit.models import AtomGitIssue, AtomGitPullRequest
32
+ from app.models.issue import Issue
33
+ from app.models.pull_request import (
34
+ PRComment,
35
+ PRCommit,
36
+ PRFile,
37
+ PRState,
38
+ PullRequest,
39
+ ReviewEvent,
40
+ ReviewEventSource,
41
+ )
42
+ from app.models.repository import (
43
+ Repository,
44
+ SyncResource,
45
+ SyncRun,
46
+ SyncStatus,
47
+ SyncTrigger,
48
+ )
49
+ from app.services import credential_service, repository_service
50
+
51
+ logger = structlog.get_logger(__name__)
52
+
53
+ # 单次同步最多拉取的页数,防止配置错误导致无限翻页
54
+ MAX_PAGES = 40
55
+
56
+
57
+ def content_hash(*parts: str | None) -> str:
58
+ """内容指纹。用于判断是否需要重新触发 AI 分析等昂贵操作。"""
59
+ digest = hashlib.sha256()
60
+ for part in parts:
61
+ digest.update((part or "").encode("utf-8"))
62
+ digest.update(b"\x00")
63
+ return digest.hexdigest()
64
+
65
+
66
+ def _pr_state(value: str) -> PRState:
67
+ try:
68
+ return PRState(value)
69
+ except ValueError:
70
+ return PRState.OPEN
71
+
72
+
73
+ def _dedupe_by(items: list, *, key) -> list:
74
+ """按 key 去重,保留首次出现。
75
+
76
+ 上游的提交列表会返回重复的 SHA —— 实测 PR #19344 的 9 条提交里有 2 条
77
+ 重复(分支被 force-push 或含合并提交时出现)。而 (pull_request_id, sha)
78
+ 上有唯一约束,直接整批插入会撞约束并让整个同步批次失败。
79
+ 保留首次出现即可:重复项指向同一个提交,信息完全相同。
80
+ """
81
+ seen: set = set()
82
+ result = []
83
+ for item in items:
84
+ value = key(item)
85
+ if value in seen:
86
+ continue
87
+ seen.add(value)
88
+ result.append(item)
89
+ return result
90
+
91
+
92
+ # ---------------------------------------------------------------------------
93
+ # Pull Request
94
+ # ---------------------------------------------------------------------------
95
+
96
+
97
+ def _pull_row(repository_id: uuid.UUID, pr: AtomGitPullRequest) -> dict:
98
+ gate = derive_gate(pr.label_names, is_draft=pr.draft, state=pr.state)
99
+ return {
100
+ "id": uuid.uuid4(),
101
+ "repository_id": repository_id,
102
+ "number": pr.number,
103
+ "title": pr.title,
104
+ "body": pr.body,
105
+ "state": _pr_state(pr.state).value,
106
+ "draft": pr.draft,
107
+ "author_login": pr.author_login,
108
+ "author_name": pr.user.name if pr.user else None,
109
+ "author_avatar_url": pr.user.avatar_url if pr.user else None,
110
+ "source_branch": pr.source_branch,
111
+ "target_branch": pr.target_branch,
112
+ "labels": [label.model_dump(mode="json") for label in pr.labels],
113
+ "label_names": pr.label_names,
114
+ "added_lines": pr.added_lines,
115
+ "removed_lines": pr.removed_lines,
116
+ "comments_count": pr.notes,
117
+ "mergeable": pr.mergeable,
118
+ "merge_conflict": pr.has_merge_conflict,
119
+ "merge_check_detail": (
120
+ pr.mergeable_state.model_dump(mode="json") if pr.mergeable_state else None
121
+ ),
122
+ "gate_stage": gate.stage.value,
123
+ "atomgit_created_at": pr.created_at,
124
+ "atomgit_updated_at": pr.updated_at,
125
+ "merged_at": pr.merged_at,
126
+ "closed_at": pr.closed_at,
127
+ "html_url": pr.permalink,
128
+ "last_synced_at": datetime.now(UTC),
129
+ "content_hash": content_hash(pr.title, pr.body, ",".join(sorted(pr.label_names))),
130
+ }
131
+
132
+
133
+ _UPSERT_UPDATE_COLUMNS = (
134
+ "title",
135
+ "body",
136
+ "state",
137
+ "draft",
138
+ "author_login",
139
+ "author_name",
140
+ "author_avatar_url",
141
+ "source_branch",
142
+ "target_branch",
143
+ "labels",
144
+ "label_names",
145
+ "added_lines",
146
+ "removed_lines",
147
+ "comments_count",
148
+ "mergeable",
149
+ "merge_conflict",
150
+ "merge_check_detail",
151
+ "gate_stage",
152
+ "atomgit_created_at",
153
+ "atomgit_updated_at",
154
+ "merged_at",
155
+ "closed_at",
156
+ "html_url",
157
+ "last_synced_at",
158
+ "content_hash",
159
+ )
160
+
161
+
162
+ async def upsert_pulls(
163
+ session: AsyncSession, repository: Repository, pulls: list[AtomGitPullRequest]
164
+ ) -> tuple[int, int]:
165
+ """写入 PR 列表,返回 (新增数, 更新数)。
166
+
167
+ 以 (repository_id, number) 为冲突键。冲突时仅当远端 updated_at
168
+ 不早于本地值才覆盖 —— 防止并发同步时用旧数据覆盖新数据。
169
+ """
170
+ if not pulls:
171
+ return 0, 0
172
+
173
+ numbers = [p.number for p in pulls]
174
+ existing = set(
175
+ (
176
+ await session.scalars(
177
+ select(PullRequest.number).where(
178
+ PullRequest.repository_id == repository.id,
179
+ PullRequest.number.in_(numbers),
180
+ )
181
+ )
182
+ ).all()
183
+ )
184
+
185
+ rows = [_pull_row(repository.id, pr) for pr in pulls]
186
+
187
+ stmt = pg_insert(PullRequest).values(rows)
188
+ update_set = {col: getattr(stmt.excluded, col) for col in _UPSERT_UPDATE_COLUMNS}
189
+ # 仅当内容指纹变化时才重新拉详情 —— 这是把详情同步量从"每轮全部 PR"
190
+ # 降到"每轮仅变化项"的关键
191
+ update_set["needs_detail"] = stmt.excluded.content_hash.is_distinct_from(
192
+ PullRequest.content_hash
193
+ )
194
+ stmt = stmt.on_conflict_do_update(
195
+ constraint="uq_ksp_pull_request_repo_number",
196
+ set_=update_set,
197
+ where=PullRequest.atomgit_updated_at.is_(None)
198
+ | (stmt.excluded.atomgit_updated_at >= PullRequest.atomgit_updated_at),
199
+ )
200
+ await session.execute(stmt)
201
+
202
+ created = sum(1 for n in numbers if n not in existing)
203
+ return created, len(numbers) - created
204
+
205
+
206
+ async def upsert_issues(
207
+ session: AsyncSession, repository: Repository, issues: list[AtomGitIssue]
208
+ ) -> tuple[int, int]:
209
+ if not issues:
210
+ return 0, 0
211
+
212
+ numbers = [i.number for i in issues]
213
+ existing = set(
214
+ (
215
+ await session.scalars(
216
+ select(Issue.number).where(
217
+ Issue.repository_id == repository.id, Issue.number.in_(numbers)
218
+ )
219
+ )
220
+ ).all()
221
+ )
222
+
223
+ rows = [
224
+ {
225
+ "id": uuid.uuid4(),
226
+ "repository_id": repository.id,
227
+ "number": issue.number,
228
+ "title": issue.title,
229
+ "body": issue.body,
230
+ "state": issue.state,
231
+ "issue_type": issue.issue_type,
232
+ "issue_state": issue.issue_state,
233
+ "priority": issue.priority,
234
+ "priority_label": issue.priority_label,
235
+ "author_login": issue.user.login if issue.user else None,
236
+ "assignee_login": issue.assignee.login if issue.assignee else None,
237
+ "labels": [label.model_dump(mode="json") for label in issue.labels],
238
+ "label_names": issue.label_names,
239
+ "comments_count": issue.comments,
240
+ "atomgit_created_at": issue.created_at,
241
+ "atomgit_updated_at": issue.updated_at,
242
+ "finished_at": issue.finished_at,
243
+ "html_url": issue.html_url,
244
+ "last_synced_at": datetime.now(UTC),
245
+ "content_hash": content_hash(issue.title, issue.body, issue.issue_state),
246
+ }
247
+ for issue in issues
248
+ ]
249
+
250
+ stmt = pg_insert(Issue).values(rows)
251
+ update_columns = (
252
+ "title",
253
+ "body",
254
+ "state",
255
+ "issue_type",
256
+ "issue_state",
257
+ "priority",
258
+ "priority_label",
259
+ "assignee_login",
260
+ "labels",
261
+ "label_names",
262
+ "comments_count",
263
+ "atomgit_updated_at",
264
+ "finished_at",
265
+ "html_url",
266
+ "last_synced_at",
267
+ "content_hash",
268
+ )
269
+ stmt = stmt.on_conflict_do_update(
270
+ constraint="uq_ksp_issue_repo_number",
271
+ set_={col: getattr(stmt.excluded, col) for col in update_columns},
272
+ where=Issue.atomgit_updated_at.is_(None)
273
+ | (stmt.excluded.atomgit_updated_at >= Issue.atomgit_updated_at),
274
+ )
275
+ await session.execute(stmt)
276
+
277
+ created = sum(1 for n in numbers if n not in existing)
278
+ return created, len(numbers) - created
279
+
280
+
281
+ # ---------------------------------------------------------------------------
282
+ # PR 详情:提交、文件、评审事件
283
+ # ---------------------------------------------------------------------------
284
+
285
+
286
+ async def sync_pull_detail(
287
+ session: AsyncSession, client: AtomGitClient, repository: Repository, pull: PullRequest
288
+ ) -> None:
289
+ """拉取单个 PR 的提交、变更文件,并解析评审事件。
290
+
291
+ 评审事件由标签与评论共同推导 —— 评审人身份只存在于评论中,
292
+ 标签仅给出汇总结果。
293
+ """
294
+ owner, name = repository.owner, repository.name
295
+
296
+ commits = await client.get_pull_commits(owner, name, pull.number)
297
+ files = await client.get_pull_files(owner, name, pull.number)
298
+ comments = await client.get_pull_comments(owner, name, pull.number)
299
+
300
+ commits = _dedupe_by(commits, key=lambda commit: commit.sha)
301
+ files = _dedupe_by(files, key=lambda file: file.filename)
302
+
303
+ # --- 提交:整批替换,避免处理"已从 PR 中移除的提交"这类脏数据 ---
304
+ await session.execute(PRCommit.__table__.delete().where(PRCommit.pull_request_id == pull.id))
305
+ session.add_all(
306
+ [
307
+ PRCommit(
308
+ pull_request_id=pull.id,
309
+ sha=commit.sha,
310
+ sequence=index,
311
+ subject=commit.subject,
312
+ message=commit.message,
313
+ author_name=(commit.commit.author.name if commit.commit.author else None),
314
+ author_email=commit.author_email,
315
+ committed_at=commit.commit.author.date if commit.commit.author else None,
316
+ signed_off_bys=_extract_signed_off_bys(commit.message),
317
+ )
318
+ for index, commit in enumerate(commits, start=1)
319
+ ]
320
+ )
321
+
322
+ # --- 变更文件:同样整批替换 ---
323
+ await session.execute(PRFile.__table__.delete().where(PRFile.pull_request_id == pull.id))
324
+ session.add_all(
325
+ [
326
+ PRFile(
327
+ pull_request_id=pull.id,
328
+ filename=file.filename,
329
+ status=file.status,
330
+ additions=file.additions,
331
+ deletions=file.deletions,
332
+ is_binary=file.is_binary,
333
+ too_large=bool(file.patch and file.patch.too_large),
334
+ diff=file.diff_text,
335
+ )
336
+ for file in files
337
+ ]
338
+ )
339
+ pull.changed_files = len(files)
340
+
341
+ # --- 评论原文:留存并展示 ---
342
+ #
343
+ # 评审意见、维护者的提问、以及 openEuler 机器人自动生成的变更摘要与
344
+ # 代码审查结论,都只存在于评论里。此前这些内容被解析成评审事件后即丢弃,
345
+ # 界面上无从看到讨论过程,维护者必须跳回 AtomGit 才能读懂评审意见。
346
+ #
347
+ # 用 upsert 而非整批替换:评论可以被编辑,但不该因为一次拉取失败就消失。
348
+ if comments:
349
+ stmt = pg_insert(PRComment).values(
350
+ [
351
+ {
352
+ "pull_request_id": pull.id,
353
+ "comment_id": str(comment.id),
354
+ "comment_type": comment.comment_type,
355
+ "discussion_id": (
356
+ str(comment.discussion_id) if comment.discussion_id else None
357
+ ),
358
+ "author_login": comment.author_login or "",
359
+ "author_avatar_url": comment.user.avatar_url if comment.user else None,
360
+ "body": comment.body or "",
361
+ "atomgit_created_at": comment.created_at,
362
+ "atomgit_updated_at": comment.updated_at,
363
+ "last_synced_at": datetime.now(UTC),
364
+ }
365
+ for comment in comments
366
+ ]
367
+ )
368
+ stmt = stmt.on_conflict_do_update(
369
+ constraint="uq_ksp_pr_comment_identity",
370
+ set_={
371
+ column: getattr(stmt.excluded, column)
372
+ for column in (
373
+ "body",
374
+ "author_login",
375
+ "author_avatar_url",
376
+ "comment_type",
377
+ "discussion_id",
378
+ "atomgit_updated_at",
379
+ "last_synced_at",
380
+ )
381
+ },
382
+ )
383
+ await session.execute(stmt)
384
+
385
+ # --- 评审事件:只持久化评论派生的部分 ---
386
+ #
387
+ # 标签代表"当前状态",评论代表"历史事实",两者语义不同:
388
+ # 标签本身没有时间戳,任何合成的时间(如用 updated_at)都会随每次同步变化,
389
+ # 导致唯一约束失效、同一标签被反复写入。而评论带有真实且稳定的时间戳,
390
+ # 且已包含评审人身份与动作时刻 —— 评审工作量与 SLA 所需的全部信息都在这里。
391
+ #
392
+ # 当前门禁状态不落库,由 derive_gate 从 label_names 实时推导(见 domain/review.py)。
393
+ events = parse_events(
394
+ labels=[],
395
+ comments=[
396
+ (comment.body, comment.author_login or "", comment.created_at)
397
+ for comment in comments
398
+ if comment.created_at is not None
399
+ ],
400
+ )
401
+ for event in events:
402
+ await session.execute(
403
+ pg_insert(ReviewEvent)
404
+ .values(
405
+ pull_request_id=pull.id,
406
+ event_type=event.event_type.value,
407
+ actor_login=event.actor_login,
408
+ source=ReviewEventSource(event.source).value,
409
+ raw_payload={"raw": event.raw},
410
+ occurred_at=event.occurred_at,
411
+ )
412
+ .on_conflict_do_nothing(constraint="uq_ksp_review_event_identity")
413
+ )
414
+
415
+ pull.detail_synced_at = datetime.now(UTC)
416
+ pull.needs_detail = False
417
+
418
+
419
+ def _extract_signed_off_bys(message: str) -> list[str]:
420
+ """提取 Signed-off-by 链。openEuler 要求该链完整。"""
421
+ result: list[str] = []
422
+ for line in message.splitlines():
423
+ stripped = line.strip()
424
+ lowered = stripped.lower()
425
+ if lowered.startswith("signed-off-by:"):
426
+ result.append(stripped.split(":", 1)[1].strip())
427
+ return result
428
+
429
+
430
+ async def pulls_needing_detail(
431
+ session: AsyncSession, repository_id: uuid.UUID, *, limit: int
432
+ ) -> list[PullRequest]:
433
+ """挑出需要拉详情的 PR:从未拉过的,或内容已变化的。
434
+
435
+ 活跃 PR(open)优先于已合并的,保证维护者看到的是当前状态。
436
+ """
437
+ stmt = (
438
+ select(PullRequest)
439
+ .where(
440
+ PullRequest.repository_id == repository_id,
441
+ PullRequest.needs_detail.is_(True),
442
+ )
443
+ .order_by(
444
+ PullRequest.detail_synced_at.is_(None).desc(),
445
+ PullRequest.state.asc(),
446
+ PullRequest.atomgit_updated_at.desc(),
447
+ )
448
+ .limit(limit)
449
+ )
450
+ return list((await session.scalars(stmt)).all())
451
+
452
+
453
+ # ---------------------------------------------------------------------------
454
+ # 同步任务编排
455
+ # ---------------------------------------------------------------------------
456
+
457
+
458
+ async def _start_run(
459
+ session: AsyncSession,
460
+ repository: Repository,
461
+ resource: SyncResource,
462
+ trigger: SyncTrigger,
463
+ ) -> SyncRun:
464
+ run = SyncRun(
465
+ repository_id=repository.id,
466
+ resource=resource,
467
+ trigger=trigger,
468
+ status=SyncStatus.RUNNING,
469
+ )
470
+ session.add(run)
471
+ await session.flush()
472
+ return run
473
+
474
+
475
+ async def _finish_run(
476
+ session: AsyncSession,
477
+ run: SyncRun,
478
+ *,
479
+ status: SyncStatus,
480
+ fetched: int = 0,
481
+ created: int = 0,
482
+ updated: int = 0,
483
+ failed: int = 0,
484
+ api_calls: int = 0,
485
+ error: str | None = None,
486
+ ) -> None:
487
+ run.status = status
488
+ run.finished_at = datetime.now(UTC)
489
+ run.items_fetched = fetched
490
+ run.items_created = created
491
+ run.items_updated = updated
492
+ run.items_failed = failed
493
+ run.api_calls = api_calls
494
+ run.error = error
495
+ await session.flush()
496
+
497
+
498
+ def _skip_without_credential(repository: Repository) -> None:
499
+ """仓库还没配凭据时跳过同步,不留下任何记录。
500
+
501
+ 这**不是**同步失败,而是"这个仓库还没启用",状态由
502
+ ``repository.credential_id`` 是否为空表达就够 —— 界面据此提示去配凭据。
503
+
504
+ 以前这里写一条 ``SyncRun(status=FAILED, error="仓库未配置凭据")``,
505
+ 两个问题:一是定时任务每 10/30/5 分钟各跑一轮,一个没配凭据的仓库一天
506
+ 能攒下几百条一模一样的失败记录,把 ``sync_runs`` 表淹掉;二是它 return
507
+ 在 ``mark_synced`` 之前,``repository.last_sync_error`` 根本没被写入 ——
508
+ 而界面上那块红框读的正是后者。等于留了噪音、却没有传递信息。
509
+ """
510
+ logger.debug("sync_skipped_without_credential", repository=repository.full_name)
511
+
512
+
513
+ async def sync_pulls(
514
+ session: AsyncSession,
515
+ settings: Settings,
516
+ repository: Repository,
517
+ *,
518
+ trigger: SyncTrigger = SyncTrigger.SCHEDULED,
519
+ states: tuple[str, ...] = ("open", "merged"),
520
+ ) -> SyncRun | None:
521
+ """同步 PR 列表。未配置凭据时返回 None(跳过,见 ``_skip_without_credential``)。
522
+
523
+ 默认同时同步 open 与 merged:维护者既要看当前待办,
524
+ 也要看近期合入情况用于统计趋势。
525
+
526
+ ``repository.backfill_days`` 生效于此:列表按更新时间倒序返回,
527
+ 一旦某页全部早于回填窗口,即可判定后续无需再拉 ——
528
+ 否则已合并 PR 有上万个,会一路翻页到安全上限。
529
+ """
530
+ if repository.credential_id is None:
531
+ _skip_without_credential(repository)
532
+ return None
533
+
534
+ run = await _start_run(session, repository, SyncResource.PULLS, trigger)
535
+ credential = await credential_service.require_credential(session, repository.credential_id)
536
+
537
+ client = repository_service.build_client(settings, repository, credential)
538
+
539
+ cutoff = datetime.now(UTC) - timedelta(days=repository.backfill_days or 180)
540
+ fetched = created = updated = 0
541
+ error: str | None = None
542
+ status = SyncStatus.SUCCEEDED
543
+
544
+ try:
545
+ async with client:
546
+ for state in states:
547
+ async for page in client.iter_pulls(
548
+ repository.owner, repository.name, state=state, per_page=100
549
+ ):
550
+ c, u = await upsert_pulls(session, repository, page.items)
551
+ fetched += len(page.items)
552
+ created += c
553
+ updated += u
554
+
555
+ if page.items and all(
556
+ item.updated_at is not None and item.updated_at < cutoff
557
+ for item in page.items
558
+ ):
559
+ # 整页都已早于回填窗口,后续只会更旧
560
+ logger.info(
561
+ "sync_pulls_backfill_window_reached",
562
+ repository=repository.full_name,
563
+ state=state,
564
+ page=page.page,
565
+ cutoff_days=repository.backfill_days,
566
+ )
567
+ break
568
+
569
+ if page.page >= MAX_PAGES:
570
+ logger.warning(
571
+ "sync_pulls_page_limit_reached",
572
+ repository=repository.full_name,
573
+ state=state,
574
+ )
575
+ status = SyncStatus.PARTIAL
576
+ break
577
+ except AtomGitError as exc:
578
+ error = str(exc)
579
+ status = SyncStatus.FAILED if fetched == 0 else SyncStatus.PARTIAL
580
+ logger.warning("sync_pulls_failed", repository=repository.full_name, error=error)
581
+
582
+ await _finish_run(
583
+ session,
584
+ run,
585
+ status=status,
586
+ fetched=fetched,
587
+ created=created,
588
+ updated=updated,
589
+ api_calls=client.stats.requests,
590
+ error=error,
591
+ )
592
+ repository_service.mark_synced(repository, error=error)
593
+ return run
594
+
595
+
596
+ async def sync_issues(
597
+ session: AsyncSession,
598
+ settings: Settings,
599
+ repository: Repository,
600
+ *,
601
+ trigger: SyncTrigger = SyncTrigger.SCHEDULED,
602
+ states: tuple[str, ...] = ("open",),
603
+ ) -> SyncRun | None:
604
+ """同步 Issue 列表。未配置凭据时返回 None(跳过,见 ``_skip_without_credential``)。"""
605
+ if repository.credential_id is None:
606
+ _skip_without_credential(repository)
607
+ return None
608
+
609
+ run = await _start_run(session, repository, SyncResource.ISSUES, trigger)
610
+ credential = await credential_service.require_credential(session, repository.credential_id)
611
+ client = repository_service.build_client(settings, repository, credential)
612
+
613
+ fetched = created = updated = 0
614
+ error: str | None = None
615
+ status = SyncStatus.SUCCEEDED
616
+
617
+ try:
618
+ async with client:
619
+ for state in states:
620
+ async for page in client.iter_issues(
621
+ repository.owner, repository.name, state=state, per_page=100
622
+ ):
623
+ c, u = await upsert_issues(session, repository, page.items)
624
+ fetched += len(page.items)
625
+ created += c
626
+ updated += u
627
+ if page.page >= MAX_PAGES:
628
+ status = SyncStatus.PARTIAL
629
+ break
630
+ except AtomGitError as exc:
631
+ error = str(exc)
632
+ status = SyncStatus.FAILED if fetched == 0 else SyncStatus.PARTIAL
633
+ logger.warning("sync_issues_failed", repository=repository.full_name, error=error)
634
+
635
+ await _finish_run(
636
+ session,
637
+ run,
638
+ status=status,
639
+ fetched=fetched,
640
+ created=created,
641
+ updated=updated,
642
+ api_calls=client.stats.requests,
643
+ error=error,
644
+ )
645
+ repository_service.mark_synced(repository, error=error)
646
+ return run
647
+
648
+
649
+ async def sync_pull_details(
650
+ session: AsyncSession,
651
+ settings: Settings,
652
+ repository: Repository,
653
+ *,
654
+ limit: int = 50,
655
+ trigger: SyncTrigger = SyncTrigger.SCHEDULED,
656
+ ) -> SyncRun | None:
657
+ """为内容有变化的 PR 补齐提交、文件与评审事件。未配置凭据时返回 None。"""
658
+ if repository.credential_id is None:
659
+ _skip_without_credential(repository)
660
+ return None
661
+
662
+ run = await _start_run(session, repository, SyncResource.PULLS, trigger)
663
+ credential = await credential_service.require_credential(session, repository.credential_id)
664
+ client = repository_service.build_client(settings, repository, credential)
665
+
666
+ targets = await pulls_needing_detail(session, repository.id, limit=limit)
667
+ succeeded = failed = 0
668
+ error: str | None = None
669
+
670
+ try:
671
+ async with client:
672
+ for pull in targets:
673
+ try:
674
+ # 每个 PR 一个保存点:写入失败时只回滚这一个 PR,
675
+ # 外层事务仍可继续。否则一次约束冲突会污染整个事务,
676
+ # 后续所有 PR 的写入都会跟着失败。
677
+ async with session.begin_nested():
678
+ await sync_pull_detail(session, client, repository, pull)
679
+ succeeded += 1
680
+ except (AtomGitError, SQLAlchemyError) as exc:
681
+ failed += 1
682
+ logger.warning(
683
+ "sync_pull_detail_failed",
684
+ repository=repository.full_name,
685
+ number=pull.number,
686
+ error=str(exc)[:300],
687
+ )
688
+ # 单个 PR 失败不阻塞整体:记录后继续,下次同步会重试
689
+ except AtomGitError as exc:
690
+ error = str(exc)
691
+
692
+ status = (
693
+ SyncStatus.FAILED
694
+ if succeeded == 0 and failed > 0
695
+ else SyncStatus.PARTIAL
696
+ if failed > 0
697
+ else SyncStatus.SUCCEEDED
698
+ )
699
+
700
+ await _finish_run(
701
+ session,
702
+ run,
703
+ status=status,
704
+ fetched=len(targets),
705
+ updated=succeeded,
706
+ failed=failed,
707
+ api_calls=client.stats.requests,
708
+ error=error,
709
+ )
710
+ return run
711
+
712
+
713
+ async def sync_repository(
714
+ session: AsyncSession,
715
+ settings: Settings,
716
+ repository: Repository,
717
+ *,
718
+ trigger: SyncTrigger = SyncTrigger.SCHEDULED,
719
+ detail_limit: int = 50,
720
+ ) -> list[SyncRun]:
721
+ """完整同步一个仓库:列表 → 详情。
722
+
723
+ 这是用户点「立即同步」走的路径,所以无凭据时**明确报错**而不是像定时
724
+ 任务那样静默跳过 —— 他正等着结果,得知道卡在哪一步。
725
+ """
726
+ if repository.credential_id is None:
727
+ raise ValidationError("仓库未配置凭据,请先在「设置 → 凭据」中添加 AtomGit token")
728
+
729
+ runs: list[SyncRun] = []
730
+ for run in (
731
+ await sync_pulls(session, settings, repository, trigger=trigger),
732
+ await sync_issues(session, settings, repository, trigger=trigger),
733
+ await sync_pull_details(
734
+ session, settings, repository, limit=detail_limit, trigger=trigger
735
+ ),
736
+ ):
737
+ # 无凭据在上面已经拦下,走不到 None 分支 —— 留着它是为了不让调用方
738
+ # 必须记住"跳过只可能发生在无凭据时"这个前提。
739
+ if run is not None:
740
+ runs.append(run)
741
+
742
+ await session.commit()
743
+ return runs
744
+
745
+
746
+ async def count_pulls_by_state(session: AsyncSession, repository_id: uuid.UUID) -> dict[str, int]:
747
+ stmt = (
748
+ select(PullRequest.state, func.count())
749
+ .where(PullRequest.repository_id == repository_id)
750
+ .group_by(PullRequest.state)
751
+ )
752
+ return {row[0]: row[1] for row in (await session.execute(stmt)).all()}