@misterhuydo/sentinel 1.5.4 → 1.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cairn/.hint-lock +1 -1
- package/.cairn/minify-map.json +13 -1
- package/.cairn/session.json +2 -2
- package/.cairn/views/23edf4_sentinel_boss.py +3664 -0
- package/.cairn/views/5f5141_main.py +1067 -0
- package/.cairn/views/62a614_bundle.js +4 -1
- package/.cairn/views/7802b9_cicd_trigger.py +171 -0
- package/.cairn/views/ac3df4_repo_task_engine.py +351 -0
- package/lib/.cairn/minify-map.json +6 -0
- package/lib/.cairn/views/2a85cc_init.js +380 -0
- package/lib/.cairn/views/e26996_slack-setup.js +97 -0
- package/lib/.cairn/views/fb78ac_upgrade.js +36 -1
- package/lib/.cairn/views/fc4a1a_add.js +164 -51
- package/lib/init.js +54 -0
- package/lib/maven.js +212 -0
- package/lib/slack-setup.js +5 -0
- package/package.json +1 -1
- package/python/requirements.txt +1 -0
- package/python/sentinel/.cairn/.cairn-project +0 -0
- package/python/sentinel/.cairn/.hint-lock +1 -0
- package/python/sentinel/.cairn/session.json +9 -0
- package/python/sentinel/__pycache__/sentinel_boss.cpython-311.pyc +0 -0
- package/python/sentinel/config_loader.py +29 -10
- package/python/sentinel/dependency_manager.py +9 -2
- package/python/sentinel/git_manager.py +23 -0
- package/python/sentinel/issue_watcher.py +7 -1
- package/python/sentinel/main.py +353 -8
- package/python/sentinel/notify.py +44 -12
- package/python/sentinel/repo_task_engine.py +49 -7
- package/python/sentinel/sentinel_boss.py +117 -3
- package/python/sentinel/slack_bot.py +15 -2
- package/python/sentinel/state_store.py +0 -1
- package/python/tests/__init__.py +0 -0
- package/python/tests/test_config_loader.py +138 -0
- package/python/tests/test_log_parser.py +62 -0
- package/python/tests/test_repo_router.py +73 -0
- package/python/tests/test_smoke.py +96 -0
- package/python/tests/test_state_store.py +128 -0
|
@@ -78,6 +78,8 @@ class SentinelConfig:
|
|
|
78
78
|
sync_max_file_mb: int = 200 # truncate synced log files exceeding this size (MB)
|
|
79
79
|
boss_mode: str = "standard" # standard | strict | fun
|
|
80
80
|
sentinel_dev_repo_path: str = "" # path to Sentinel source repo for Dev Claude
|
|
81
|
+
sentinel_dev_soak_minutes: int = 30 # minutes to monitor after a Patch restart before publishing
|
|
82
|
+
sentinel_dev_auto_publish: bool = False # if True, auto-publish to npm after clean soak
|
|
81
83
|
|
|
82
84
|
|
|
83
85
|
@dataclass
|
|
@@ -112,8 +114,13 @@ class RepoConfig:
|
|
|
112
114
|
cicd_user: str = "" # Jenkins username for Basic auth (defaults to "sentinel")
|
|
113
115
|
health_url: str = "" # optional: HTTP endpoint returning {"Status": "true"}
|
|
114
116
|
cicd_token: str = ""
|
|
115
|
-
|
|
116
|
-
|
|
117
|
+
git_ssh_user_key: str = "" # personal GitHub SSH key (contributor on org repos)
|
|
118
|
+
git_ssh_deploy_key: str = "" # repo-specific deploy key (repos you own/admin)
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def ssh_key_file(self) -> str:
|
|
122
|
+
"""Resolved SSH key — deploy key takes priority over user key."""
|
|
123
|
+
return self.git_ssh_deploy_key or self.git_ssh_user_key
|
|
117
124
|
|
|
118
125
|
|
|
119
126
|
# ── Loader ────────────────────────────────────────────────────────────────────
|
|
@@ -123,6 +130,7 @@ class ConfigLoader:
|
|
|
123
130
|
self.config_dir = Path(config_dir)
|
|
124
131
|
self.sentinel: SentinelConfig = SentinelConfig()
|
|
125
132
|
self.log_sources: dict[str, LogSourceConfig] = {}
|
|
133
|
+
self._on_reload_callbacks: list = [] # called after every load()
|
|
126
134
|
self.repos: dict[str, RepoConfig] = {}
|
|
127
135
|
self.load()
|
|
128
136
|
self._register_sighup()
|
|
@@ -135,6 +143,15 @@ class ConfigLoader:
|
|
|
135
143
|
"Config loaded: %d log-config(s), %d repo-config(s)",
|
|
136
144
|
len(self.log_sources), len(self.repos),
|
|
137
145
|
)
|
|
146
|
+
for cb in self._on_reload_callbacks:
|
|
147
|
+
try:
|
|
148
|
+
cb(self)
|
|
149
|
+
except Exception as e:
|
|
150
|
+
logger.warning("Config reload callback failed: %s", e)
|
|
151
|
+
|
|
152
|
+
def register_on_reload(self, callback) -> None:
|
|
153
|
+
"""Register a callback(cfg_loader) to be called after every config reload."""
|
|
154
|
+
self._on_reload_callbacks.append(callback)
|
|
138
155
|
|
|
139
156
|
def _load_sentinel(self):
|
|
140
157
|
# Load workspace-level config first (~/sentinel/sentinel.properties),
|
|
@@ -218,6 +235,8 @@ class ConfigLoader:
|
|
|
218
235
|
raw_mode = d.get("BOSS_MODE", "standard").lower().strip()
|
|
219
236
|
c.boss_mode = raw_mode if raw_mode in ("standard", "strict", "fun") else "standard"
|
|
220
237
|
c.sentinel_dev_repo_path = d.get("SENTINEL_DEV_REPO_PATH", "")
|
|
238
|
+
c.sentinel_dev_soak_minutes = int(d.get("SENTINEL_DEV_SOAK_MINUTES", 30))
|
|
239
|
+
c.sentinel_dev_auto_publish = d.get("SENTINEL_DEV_AUTO_PUBLISH", "false").lower() == "true"
|
|
221
240
|
self.sentinel = c
|
|
222
241
|
|
|
223
242
|
def _load_log_sources(self):
|
|
@@ -254,8 +273,8 @@ class ConfigLoader:
|
|
|
254
273
|
# Project-level defaults from sentinel.properties
|
|
255
274
|
sentinel_path = self.config_dir / "sentinel.properties"
|
|
256
275
|
proj_d: dict[str, str] = _parse_properties(str(sentinel_path)) if sentinel_path.exists() else {}
|
|
257
|
-
|
|
258
|
-
|
|
276
|
+
default_user_key = os.path.expanduser(proj_d.get("GIT_SSH_USER_KEY", ""))
|
|
277
|
+
default_deploy_key = os.path.expanduser(proj_d.get("GIT_SSH_DEPLOY_KEY", ""))
|
|
259
278
|
|
|
260
279
|
self.repos = {}
|
|
261
280
|
for path in sorted(repos_dir.glob("*.properties")):
|
|
@@ -273,15 +292,15 @@ class ConfigLoader:
|
|
|
273
292
|
r.cicd_user = d.get("CICD_USER", "")
|
|
274
293
|
r.cicd_token = d.get("CICD_TOKEN", "")
|
|
275
294
|
r.health_url = d.get("HEALTH_URL", "")
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
r.
|
|
295
|
+
raw_user_key = d.get("GIT_SSH_USER_KEY", "")
|
|
296
|
+
raw_deploy_key = d.get("GIT_SSH_DEPLOY_KEY", "") or d.get("GIT_SSH_KEY", "") or d.get("SSH_KEY_FILE", "")
|
|
297
|
+
r.git_ssh_user_key = os.path.expanduser(raw_user_key) if raw_user_key else default_user_key
|
|
298
|
+
r.git_ssh_deploy_key = os.path.expanduser(raw_deploy_key) if raw_deploy_key else default_deploy_key
|
|
280
299
|
# Auto-discover deploy key in project dir if no key configured
|
|
281
|
-
if not r.
|
|
300
|
+
if not r.git_ssh_deploy_key and not r.git_ssh_user_key:
|
|
282
301
|
auto_key = Path(self.config_dir).parent / f"{r.repo_name}.key"
|
|
283
302
|
if auto_key.exists():
|
|
284
|
-
r.
|
|
303
|
+
r.git_ssh_deploy_key = str(auto_key.resolve())
|
|
285
304
|
self.repos[r.repo_name] = r
|
|
286
305
|
|
|
287
306
|
def _register_sighup(self):
|
|
@@ -232,8 +232,8 @@ def execute_cascade(
|
|
|
232
232
|
Returns one CascadeResult per dependent repo attempted.
|
|
233
233
|
"""
|
|
234
234
|
# Import here to avoid circular imports
|
|
235
|
-
from .git_manager import commit_file_change, push_dep_update, open_dep_pr, _git_env, _git, maven_compile_check, MissingToolError # noqa: E501
|
|
236
|
-
from .notify import notify_cascade_build_failed
|
|
235
|
+
from .git_manager import commit_file_change, push_dep_update, open_dep_pr, _git_env, _git, maven_compile_check, MissingToolError, MavenAuthError # noqa: E501
|
|
236
|
+
from .notify import notify_cascade_build_failed, notify_nexus_auth_failure
|
|
237
237
|
|
|
238
238
|
all_dependents = find_dependents(artifact_id, repos, skip_repo=source_repo_name)
|
|
239
239
|
if target_repo_names:
|
|
@@ -264,6 +264,13 @@ def execute_cascade(
|
|
|
264
264
|
# Dry-run: compile before committing — catch broken deps early
|
|
265
265
|
try:
|
|
266
266
|
compile_ok, compile_output = maven_compile_check(repo.local_path)
|
|
267
|
+
except MavenAuthError as e:
|
|
268
|
+
_git(["checkout", "pom.xml"], cwd=repo.local_path, env=_git_env(repo))
|
|
269
|
+
result.error = "Maven auth failed — Nexus credentials missing or invalid"
|
|
270
|
+
logger.error("Cascade auth failure for %s: %s", repo.repo_name, e.output[-300:])
|
|
271
|
+
notify_nexus_auth_failure(cfg, repo.repo_name, f"cascade dep update {artifact_id}→{new_version}", e.output)
|
|
272
|
+
results.append(result)
|
|
273
|
+
continue
|
|
267
274
|
except MissingToolError:
|
|
268
275
|
compile_ok, compile_output = False, "mvn not installed on this server"
|
|
269
276
|
|
|
@@ -29,6 +29,25 @@ class MissingToolError(Exception):
|
|
|
29
29
|
super().__init__(f"Build tool '{tool}' not found on this server")
|
|
30
30
|
self.tool = tool
|
|
31
31
|
|
|
32
|
+
|
|
33
|
+
class MavenAuthError(Exception):
|
|
34
|
+
"""Raised when mvn fails due to Nexus/Artifactory credential issues (401/403)."""
|
|
35
|
+
def __init__(self, output: str):
|
|
36
|
+
super().__init__("Maven dependency resolution failed — Nexus credentials missing or invalid")
|
|
37
|
+
self.output = output
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_NEXUS_AUTH_RE = re.compile(
|
|
41
|
+
r"(DependencyResolutionException"
|
|
42
|
+
r"|Could not transfer artifact"
|
|
43
|
+
r"|RepoAuthenticationException"
|
|
44
|
+
r"|status code: 40[13]"
|
|
45
|
+
r"|401 Unauthorized"
|
|
46
|
+
r"|403 Forbidden"
|
|
47
|
+
r"|authentication failed)",
|
|
48
|
+
re.IGNORECASE,
|
|
49
|
+
)
|
|
50
|
+
|
|
32
51
|
# Files that must never be modified by Sentinel
|
|
33
52
|
_PROTECTED_PATHS = {".github/", "Jenkinsfile"}
|
|
34
53
|
|
|
@@ -69,6 +88,8 @@ def maven_compile_check(local_path: str, timeout: int = 300) -> tuple[bool, str]
|
|
|
69
88
|
timeout=timeout,
|
|
70
89
|
)
|
|
71
90
|
output = (r.stdout + r.stderr).strip()
|
|
91
|
+
if r.returncode != 0 and _NEXUS_AUTH_RE.search(output):
|
|
92
|
+
raise MavenAuthError(output)
|
|
72
93
|
return r.returncode == 0, output
|
|
73
94
|
|
|
74
95
|
|
|
@@ -160,6 +181,8 @@ def _run_tests(repo: RepoConfig, local_path: str) -> bool:
|
|
|
160
181
|
if r.returncode != 0:
|
|
161
182
|
output = r.stdout[-3000:] + r.stderr[-1000:]
|
|
162
183
|
logger.error("Tests failed:\n%s", output)
|
|
184
|
+
if _NEXUS_AUTH_RE.search(output):
|
|
185
|
+
raise MavenAuthError(output)
|
|
163
186
|
# Detect missing sub-tool invoked by the build (e.g. yarn called from Maven exec plugin)
|
|
164
187
|
import re as _re
|
|
165
188
|
m = _re.search(r'Cannot run program "([^"]+)"', output)
|
|
@@ -42,6 +42,7 @@ class IssueEvent:
|
|
|
42
42
|
severity: str = "ERROR"
|
|
43
43
|
timestamp: str = ""
|
|
44
44
|
submitter_user_id: str = "" # Slack user ID who raised this via Boss, if known
|
|
45
|
+
origin_channel: str = "" # Slack channel where the issue was raised, if known
|
|
45
46
|
|
|
46
47
|
# Compatibility fields matching ErrorEvent interface
|
|
47
48
|
level: str = "ERROR"
|
|
@@ -130,8 +131,9 @@ def scan_issues(project_dir: Path) -> list[IssueEvent]:
|
|
|
130
131
|
|
|
131
132
|
# Parse metadata headers in any order (TARGET_REPO, SUBMITTED_BY, SUBMITTED_AT, etc.)
|
|
132
133
|
import re as _re
|
|
133
|
-
_META = ("TARGET_REPO:", "SUBMITTED_BY:", "SUBMITTED_AT:", "SUPPORT_URL:")
|
|
134
|
+
_META = ("TARGET_REPO:", "SUBMITTED_BY:", "SUBMITTED_AT:", "SUPPORT_URL:", "ORIGIN_CHANNEL:")
|
|
134
135
|
submitter_user_id = ""
|
|
136
|
+
origin_channel = ""
|
|
135
137
|
for i, line in enumerate(lines):
|
|
136
138
|
stripped = line.strip()
|
|
137
139
|
upper = stripped.upper()
|
|
@@ -143,6 +145,9 @@ def scan_issues(project_dir: Path) -> list[IssueEvent]:
|
|
|
143
145
|
if m:
|
|
144
146
|
submitter_user_id = m.group(1)
|
|
145
147
|
body_start = i + 1
|
|
148
|
+
elif upper.startswith("ORIGIN_CHANNEL:"):
|
|
149
|
+
origin_channel = stripped[len("ORIGIN_CHANNEL:"):].strip()
|
|
150
|
+
body_start = i + 1
|
|
146
151
|
elif any(upper.startswith(p) for p in _META) or not stripped:
|
|
147
152
|
body_start = i + 1
|
|
148
153
|
else:
|
|
@@ -158,6 +163,7 @@ def scan_issues(project_dir: Path) -> list[IssueEvent]:
|
|
|
158
163
|
body=body,
|
|
159
164
|
target_repo=target_repo,
|
|
160
165
|
submitter_user_id=submitter_user_id,
|
|
166
|
+
origin_channel=origin_channel,
|
|
161
167
|
))
|
|
162
168
|
logger.info("Found issue: %s (target_repo=%r)", f.name, target_repo or "auto")
|
|
163
169
|
|
package/python/sentinel/main.py
CHANGED
|
@@ -23,7 +23,7 @@ from pathlib import Path
|
|
|
23
23
|
from .cairn_client import ensure_installed as cairn_installed, index_repo
|
|
24
24
|
from .config_loader import ConfigLoader, SentinelConfig
|
|
25
25
|
from .fix_engine import generate_fix
|
|
26
|
-
from .git_manager import apply_and_commit, publish, _git_env, MissingToolError, poll_open_prs
|
|
26
|
+
from .git_manager import apply_and_commit, publish, _git_env, MissingToolError, MavenAuthError, poll_open_prs
|
|
27
27
|
from .cicd_trigger import trigger as cicd_trigger
|
|
28
28
|
from .log_fetcher import fetch_all
|
|
29
29
|
from .log_parser import parse_all, scan_all_for_markers, ErrorEvent
|
|
@@ -223,6 +223,12 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
|
|
|
223
223
|
|
|
224
224
|
try:
|
|
225
225
|
commit_status, commit_hash = apply_and_commit(event, patch_path, repo, sentinel)
|
|
226
|
+
except MavenAuthError as e:
|
|
227
|
+
from .notify import notify_nexus_auth_failure
|
|
228
|
+
logger.error("Nexus auth failure applying fix for %s: %s", event.fingerprint, str(e))
|
|
229
|
+
notify_nexus_auth_failure(sentinel, repo.repo_name, f"fix {event.fingerprint[:8]}", e.output)
|
|
230
|
+
store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
|
|
231
|
+
return
|
|
226
232
|
except MissingToolError as e:
|
|
227
233
|
logger.warning("Missing tool for %s: %s", event.source, e)
|
|
228
234
|
if _auto_install_if_safe(e.tool, repo.local_path, sentinel, repo.repo_name, event.source):
|
|
@@ -319,13 +325,14 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
|
|
|
319
325
|
# Post "working on" to channel and capture thread_ts for progress replies
|
|
320
326
|
from .notify import slack_alert as _slack_alert, slack_thread_reply as _slack_reply
|
|
321
327
|
_submitter = getattr(event, "submitter_user_id", "")
|
|
328
|
+
_origin_channel = getattr(event, "origin_channel", "") or sentinel.slack_channel
|
|
322
329
|
_started_msg = (
|
|
323
330
|
f":hammer: Working on *<@{_submitter}>*'s request — *{repo.repo_name}*\n"
|
|
324
331
|
f"_{event.message[:120]}_"
|
|
325
332
|
) if _submitter else (
|
|
326
333
|
f":hammer: Working on *{repo.repo_name}*\n_{event.message[:120]}_"
|
|
327
334
|
)
|
|
328
|
-
_thread_ts = _slack_alert(sentinel.slack_bot_token,
|
|
335
|
+
_thread_ts = _slack_alert(sentinel.slack_bot_token, _origin_channel, _started_msg)
|
|
329
336
|
|
|
330
337
|
def _progress(msg: str) -> None:
|
|
331
338
|
"""Post a threaded reply under the 'Working on' message."""
|
|
@@ -402,7 +409,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
|
|
|
402
409
|
_progress(f":arrow_right: <{pr_url}|PR opened> — awaiting review")
|
|
403
410
|
notify_fix_applied(sentinel, event.source, event.message,
|
|
404
411
|
repo_name=repo.repo_name, branch=branch, pr_url=pr_url,
|
|
405
|
-
submitter_user_id=submitter_uid)
|
|
412
|
+
submitter_user_id=submitter_uid, origin_channel=_origin_channel)
|
|
406
413
|
mark_done(event.issue_file)
|
|
407
414
|
|
|
408
415
|
if repo.auto_publish:
|
|
@@ -415,6 +422,16 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
|
|
|
415
422
|
return {"submitter": submitter_uid, "repo_name": repo.repo_name,
|
|
416
423
|
"status": "done", "summary": event.message[:120], "pr_url": pr_url}
|
|
417
424
|
|
|
425
|
+
except MavenAuthError as e:
|
|
426
|
+
from .notify import notify_nexus_auth_failure
|
|
427
|
+
submitter_uid = getattr(event, "submitter_user_id", "")
|
|
428
|
+
logger.error("Nexus auth failure for issue %s: %s", event.fingerprint, str(e))
|
|
429
|
+
_progress(":lock: Maven authentication failed — DMing admin for Nexus credentials")
|
|
430
|
+
notify_nexus_auth_failure(sentinel, repo.repo_name, f"issue fix {event.fingerprint[:8]}", e.output)
|
|
431
|
+
store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
|
|
432
|
+
mark_done(event.issue_file)
|
|
433
|
+
return {"submitter": submitter_uid, "repo_name": repo.repo_name,
|
|
434
|
+
"status": "blocked", "summary": "Nexus credentials needed", "pr_url": ""}
|
|
418
435
|
except MissingToolError as e:
|
|
419
436
|
logger.warning("Missing tool for %s: %s", event.source, e)
|
|
420
437
|
submitter_uid = getattr(event, "submitter_user_id", "")
|
|
@@ -424,7 +441,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
|
|
|
424
441
|
)
|
|
425
442
|
if not installed:
|
|
426
443
|
_progress(f":x: `{e.tool}` is not a known safe tool — manual install required")
|
|
427
|
-
notify_missing_tool(sentinel, e.tool, repo.repo_name, event.source, submitter_uid)
|
|
444
|
+
notify_missing_tool(sentinel, e.tool, repo.repo_name, event.source, submitter_uid, _origin_channel)
|
|
428
445
|
store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
|
|
429
446
|
mark_done(event.issue_file)
|
|
430
447
|
return {"submitter": submitter_uid, "repo_name": repo.repo_name,
|
|
@@ -448,7 +465,8 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
|
|
|
448
465
|
_progress(":x: Tests still failing after tool install — needs human review")
|
|
449
466
|
notify_fix_blocked(sentinel, event.source, event.message,
|
|
450
467
|
reason="Patch was generated but commit/tests failed after tool install",
|
|
451
|
-
repo_name=repo.repo_name, submitter_user_id=submitter_uid
|
|
468
|
+
repo_name=repo.repo_name, submitter_user_id=submitter_uid,
|
|
469
|
+
origin_channel=_origin_channel)
|
|
452
470
|
mark_done(event.issue_file)
|
|
453
471
|
return {"submitter": submitter_uid, "repo_name": repo.repo_name,
|
|
454
472
|
"status": "blocked", "summary": "Commit/tests failed after tool install", "pr_url": ""}
|
|
@@ -472,7 +490,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
|
|
|
472
490
|
_progress(f":arrow_right: <{pr_url}|PR opened> — awaiting review")
|
|
473
491
|
notify_fix_applied(sentinel, event.source, event.message,
|
|
474
492
|
repo_name=repo.repo_name, branch=branch, pr_url=pr_url,
|
|
475
|
-
submitter_user_id=submitter_uid)
|
|
493
|
+
submitter_user_id=submitter_uid, origin_channel=_origin_channel)
|
|
476
494
|
mark_done(event.issue_file)
|
|
477
495
|
if repo.auto_publish:
|
|
478
496
|
ok = cicd_trigger(repo, store, event.fingerprint)
|
|
@@ -721,9 +739,125 @@ async def _startup_checks(cfg_loader: ConfigLoader) -> dict:
|
|
|
721
739
|
results["ssh"].append({"name": src_name, "host": host,
|
|
722
740
|
"status": "error", "message": str(e)})
|
|
723
741
|
|
|
742
|
+
# ── Maven settings.xml check ─────────────────────────────────────────────
|
|
743
|
+
_check_maven_settings(cfg_loader, results)
|
|
744
|
+
|
|
724
745
|
return results
|
|
725
746
|
|
|
726
747
|
|
|
748
|
+
def _check_maven_settings(cfg_loader, results: dict) -> None:
|
|
749
|
+
"""
|
|
750
|
+
Scan pom.xml files in all cloned repos for private Nexus <repository> entries.
|
|
751
|
+
Groups missing credentials by hostname and DMs the first configured admin user
|
|
752
|
+
with two options: paste full settings.xml content, or provide creds per host.
|
|
753
|
+
"""
|
|
754
|
+
import re as _re
|
|
755
|
+
import os as _os
|
|
756
|
+
from pathlib import Path as _Path
|
|
757
|
+
from urllib.parse import urlparse as _urlparse
|
|
758
|
+
|
|
759
|
+
settings_path = _Path(_os.path.expanduser("~/.m2/settings.xml"))
|
|
760
|
+
|
|
761
|
+
_PUBLIC = _re.compile(r"repo1\.maven\.org|central\.maven\.org|repo\.maven\.apache\.org", _re.I)
|
|
762
|
+
_REPO_RE = _re.compile(
|
|
763
|
+
r"<(?:repository|snapshotRepository|pluginRepository)>"
|
|
764
|
+
r"\s*<id>([^<]+)</id>\s*<url>([^<]+)</url>",
|
|
765
|
+
_re.S,
|
|
766
|
+
)
|
|
767
|
+
_SERVER_RE = _re.compile(r"<server>\s*<id>([^<]+)</id>", _re.S)
|
|
768
|
+
|
|
769
|
+
# Scan all cloned repos — group by hostname → set of repo IDs
|
|
770
|
+
nexus_map: dict[str, set[str]] = {}
|
|
771
|
+
for repo in cfg_loader.repos.values():
|
|
772
|
+
local = _Path(repo.local_path)
|
|
773
|
+
if not local.exists():
|
|
774
|
+
continue
|
|
775
|
+
for pom in local.rglob("pom.xml"):
|
|
776
|
+
if any(p in ("target", ".git", "node_modules") for p in pom.parts):
|
|
777
|
+
continue
|
|
778
|
+
try:
|
|
779
|
+
text = pom.read_text(encoding="utf-8", errors="replace")
|
|
780
|
+
except OSError:
|
|
781
|
+
continue
|
|
782
|
+
for m in _REPO_RE.finditer(text):
|
|
783
|
+
repo_id = m.group(1).strip()
|
|
784
|
+
url = m.group(2).strip()
|
|
785
|
+
if _PUBLIC.search(url):
|
|
786
|
+
continue
|
|
787
|
+
try:
|
|
788
|
+
hostname = _urlparse(url).hostname or ""
|
|
789
|
+
except Exception:
|
|
790
|
+
continue
|
|
791
|
+
if hostname:
|
|
792
|
+
nexus_map.setdefault(hostname, set()).add(repo_id)
|
|
793
|
+
|
|
794
|
+
if not nexus_map:
|
|
795
|
+
return
|
|
796
|
+
|
|
797
|
+
# Read configured server IDs from existing settings.xml
|
|
798
|
+
configured_ids: set[str] = set()
|
|
799
|
+
if settings_path.exists():
|
|
800
|
+
try:
|
|
801
|
+
for m in _SERVER_RE.finditer(settings_path.read_text(encoding="utf-8", errors="replace")):
|
|
802
|
+
configured_ids.add(m.group(1).strip())
|
|
803
|
+
except OSError:
|
|
804
|
+
pass
|
|
805
|
+
|
|
806
|
+
# Filter to hosts with at least one unconfigured ID
|
|
807
|
+
missing_hosts: dict[str, set[str]] = {
|
|
808
|
+
host: ids - configured_ids
|
|
809
|
+
for host, ids in nexus_map.items()
|
|
810
|
+
if ids - configured_ids
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
if not missing_hosts:
|
|
814
|
+
all_ids = {rid for ids in nexus_map.values() for rid in ids}
|
|
815
|
+
logger.info("Maven settings OK — %d repo ID(s) configured", len(all_ids))
|
|
816
|
+
return
|
|
817
|
+
|
|
818
|
+
summary = "; ".join(
|
|
819
|
+
f"{host} ({', '.join(sorted(ids))})"
|
|
820
|
+
for host, ids in sorted(missing_hosts.items())
|
|
821
|
+
)
|
|
822
|
+
results["warnings"].append(f"Maven credentials missing: {summary}")
|
|
823
|
+
logger.warning("Maven settings gap: %s", summary)
|
|
824
|
+
|
|
825
|
+
# DM the first admin user with two resolution options
|
|
826
|
+
cfg = cfg_loader.sentinel
|
|
827
|
+
if not cfg.slack_admin_users or not cfg.slack_bot_token:
|
|
828
|
+
logger.warning(
|
|
829
|
+
"Cannot DM admin about Maven credentials — "
|
|
830
|
+
"set SLACK_ADMIN_USERS and SLACK_BOT_TOKEN in sentinel.properties"
|
|
831
|
+
)
|
|
832
|
+
return
|
|
833
|
+
|
|
834
|
+
lines = [
|
|
835
|
+
":warning: *Maven credentials needed* — I found Nexus servers missing from `~/.m2/settings.xml`:",
|
|
836
|
+
]
|
|
837
|
+
for host, ids in sorted(missing_hosts.items()):
|
|
838
|
+
lines.append(f" • `{host}` → repo IDs: `{', '.join(sorted(ids))}`")
|
|
839
|
+
lines += [
|
|
840
|
+
"",
|
|
841
|
+
"Pick one of these two ways to fix it:",
|
|
842
|
+
"",
|
|
843
|
+
"*Option 1 — paste your existing `settings.xml`*",
|
|
844
|
+
"Copy the full content of your `~/.m2/settings.xml` and send it to me starting with:",
|
|
845
|
+
"`nexus settings`",
|
|
846
|
+
"then the XML on the next line(s). I'll extract the credentials and save them.",
|
|
847
|
+
"",
|
|
848
|
+
"*Option 2 — provide credentials per server*",
|
|
849
|
+
"Send one message per Nexus host:",
|
|
850
|
+
"`nexus creds <host> <username> <password>`",
|
|
851
|
+
"",
|
|
852
|
+
"Examples:",
|
|
853
|
+
]
|
|
854
|
+
for host in sorted(missing_hosts):
|
|
855
|
+
lines.append(f"`nexus creds {host} myuser mypassword`")
|
|
856
|
+
|
|
857
|
+
from .notify import slack_dm as _slack_dm
|
|
858
|
+
_slack_dm(cfg.slack_bot_token, cfg.slack_admin_users[0], "\n".join(lines))
|
|
859
|
+
|
|
860
|
+
|
|
727
861
|
async def _send_startup_email_delayed(cfg, results: dict, delay: int = 300):
|
|
728
862
|
"""Wait delay seconds then send startup summary email."""
|
|
729
863
|
await asyncio.sleep(delay)
|
|
@@ -1010,8 +1144,73 @@ async def _handle_dev_task(task, cfg_loader: ConfigLoader, store: StateStore):
|
|
|
1010
1144
|
if status == "done":
|
|
1011
1145
|
_slack_alert(
|
|
1012
1146
|
sentinel.slack_bot_token, sentinel.slack_channel,
|
|
1013
|
-
f"{mentions}:white_check_mark: *Patch finished* —
|
|
1147
|
+
f"{mentions}:white_check_mark: *Patch finished* — running tests before restart...",
|
|
1014
1148
|
)
|
|
1149
|
+
# Run test suite before restarting — revert if tests fail.
|
|
1150
|
+
import os as _os, sys as _sys, subprocess as _sp
|
|
1151
|
+
_loop2 = asyncio.get_event_loop()
|
|
1152
|
+
_code_dir = Path(sentinel.sentinel_dev_repo_path or ".")
|
|
1153
|
+
_venv_pytest = _code_dir / ".venv" / "bin" / "pytest"
|
|
1154
|
+
_pytest_bin = str(_venv_pytest) if _venv_pytest.exists() else "pytest"
|
|
1155
|
+
try:
|
|
1156
|
+
_test_result = await _loop2.run_in_executor(
|
|
1157
|
+
None,
|
|
1158
|
+
lambda: _sp.run(
|
|
1159
|
+
[_pytest_bin, "tests/", "-q", "--tb=short", "--no-header"],
|
|
1160
|
+
cwd=str(_code_dir), capture_output=True, text=True, timeout=120,
|
|
1161
|
+
),
|
|
1162
|
+
)
|
|
1163
|
+
except Exception as _e:
|
|
1164
|
+
_slack_alert(
|
|
1165
|
+
sentinel.slack_bot_token, sentinel.slack_channel,
|
|
1166
|
+
f"{mentions}:warning: *Patch* — could not run tests: {_e}. Restarting anyway.",
|
|
1167
|
+
)
|
|
1168
|
+
_test_result = None
|
|
1169
|
+
|
|
1170
|
+
if _test_result is not None and _test_result.returncode != 0:
|
|
1171
|
+
# Tests failed — revert the fix and stay on old code
|
|
1172
|
+
_revert = await _loop2.run_in_executor(
|
|
1173
|
+
None,
|
|
1174
|
+
lambda: _sp.run(
|
|
1175
|
+
["git", "revert", "--no-edit", "HEAD"],
|
|
1176
|
+
cwd=str(_code_dir), capture_output=True, text=True, timeout=60,
|
|
1177
|
+
),
|
|
1178
|
+
)
|
|
1179
|
+
_revert_ok = _revert.returncode == 0
|
|
1180
|
+
_test_summary = (_test_result.stdout + _test_result.stderr)[-800:]
|
|
1181
|
+
_slack_alert(
|
|
1182
|
+
sentinel.slack_bot_token, sentinel.slack_channel,
|
|
1183
|
+
f"{mentions}:x: *Patch tests failed* — fix reverted{'✓' if _revert_ok else ' (revert failed — check manually)'}.\n```{_test_summary}```",
|
|
1184
|
+
)
|
|
1185
|
+
logger.error("Patch tests failed — reverted. Output:\n%s", _test_summary)
|
|
1186
|
+
else:
|
|
1187
|
+
_summary = ""
|
|
1188
|
+
if _test_result:
|
|
1189
|
+
_lines = _test_result.stdout.strip().splitlines()
|
|
1190
|
+
_summary = f" ({_lines[-1]})" if _lines else ""
|
|
1191
|
+
_slack_alert(
|
|
1192
|
+
sentinel.slack_bot_token, sentinel.slack_channel,
|
|
1193
|
+
f"{mentions}:white_check_mark: *Patch applied* — tests passed{_summary}. Restarting...",
|
|
1194
|
+
)
|
|
1195
|
+
logger.info("Patch tests passed — restarting Sentinel (execv)")
|
|
1196
|
+
# Write soak marker so the next boot knows to monitor for regressions.
|
|
1197
|
+
_patch_hash = _sp.run(
|
|
1198
|
+
["git", "rev-parse", "HEAD"],
|
|
1199
|
+
cwd=str(_code_dir), capture_output=True, text=True,
|
|
1200
|
+
).stdout.strip()
|
|
1201
|
+
from datetime import datetime as _dt, timezone as _tz
|
|
1202
|
+
_soak_mins = cfg_loader.sentinel.sentinel_dev_soak_minutes
|
|
1203
|
+
_soak_until = (_dt.now(_tz.utc).timestamp() + _soak_mins * 60)
|
|
1204
|
+
_auto_pub = cfg_loader.sentinel.sentinel_dev_auto_publish
|
|
1205
|
+
(_code_dir / ".patch-soak").write_text(
|
|
1206
|
+
f"PATCH_HASH={_patch_hash}\n"
|
|
1207
|
+
f"PATCHED_AT={_dt.now(_tz.utc).isoformat()}\n"
|
|
1208
|
+
f"SOAK_UNTIL={_soak_until}\n"
|
|
1209
|
+
f"AUTO_PUBLISH={str(_auto_pub).lower()}\n"
|
|
1210
|
+
f"SOAK_MINUTES={_soak_mins}\n"
|
|
1211
|
+
)
|
|
1212
|
+
await asyncio.sleep(1) # let the Slack message flush
|
|
1213
|
+
_os.execv(_sys.executable, [_sys.executable] + _sys.argv)
|
|
1015
1214
|
elif status == "needs_human":
|
|
1016
1215
|
# Boss qualifies the raw Patch explanation before surfacing to users
|
|
1017
1216
|
qualified = _boss_qualify_dev_reason(detail, sentinel)
|
|
@@ -1128,7 +1327,13 @@ async def _handle_repo_task(task, repo_cfg, cfg_loader: ConfigLoader, store: Sta
|
|
|
1128
1327
|
mentions = (mentions + " ") if mentions else ""
|
|
1129
1328
|
|
|
1130
1329
|
if status == "done":
|
|
1131
|
-
if detail
|
|
1330
|
+
if detail and detail.startswith("__cicd__"):
|
|
1331
|
+
cicd_name = detail[len("__cicd__"):]
|
|
1332
|
+
_slack_alert(
|
|
1333
|
+
sentinel.slack_bot_token, sentinel.slack_channel,
|
|
1334
|
+
f"{mentions}:white_check_mark: Done — pushed to `{task.repo_name}/{repo_cfg.branch}` and triggered `{cicd_name}` release.",
|
|
1335
|
+
)
|
|
1336
|
+
elif detail: # PR URL
|
|
1132
1337
|
_slack_alert(
|
|
1133
1338
|
sentinel.slack_bot_token, sentinel.slack_channel,
|
|
1134
1339
|
f"{mentions}:white_check_mark: Done — PR opened for `{task.repo_name}`: {detail}",
|
|
@@ -1224,6 +1429,139 @@ def _log_auth_status(cfg: SentinelConfig) -> None:
|
|
|
1224
1429
|
slack_alert(cfg.slack_bot_token, cfg.slack_channel, msg)
|
|
1225
1430
|
|
|
1226
1431
|
|
|
1432
|
+
async def _patch_soak_monitor(cfg_loader: ConfigLoader) -> None:
|
|
1433
|
+
"""
|
|
1434
|
+
After a Patch-triggered restart, monitor sentinel.log for N minutes.
|
|
1435
|
+
- Clean soak → notify Slack; if AUTO_PUBLISH=true → npm publish
|
|
1436
|
+
- Crash detected → git revert HEAD, restart, alert Slack
|
|
1437
|
+
"""
|
|
1438
|
+
import subprocess as _sp, time as _time
|
|
1439
|
+
from datetime import datetime as _dt, timezone as _tz
|
|
1440
|
+
from .notify import slack_alert as _slack_alert
|
|
1441
|
+
|
|
1442
|
+
code_dir = Path(cfg_loader.sentinel.sentinel_dev_repo_path)
|
|
1443
|
+
marker = code_dir / ".patch-soak"
|
|
1444
|
+
if not marker.exists():
|
|
1445
|
+
return
|
|
1446
|
+
|
|
1447
|
+
raw = {}
|
|
1448
|
+
for line in marker.read_text().splitlines():
|
|
1449
|
+
if "=" in line:
|
|
1450
|
+
k, _, v = line.partition("=")
|
|
1451
|
+
raw[k.strip()] = v.strip()
|
|
1452
|
+
|
|
1453
|
+
patch_hash = raw.get("PATCH_HASH", "unknown")
|
|
1454
|
+
soak_until = float(raw.get("SOAK_UNTIL", 0))
|
|
1455
|
+
auto_publish = raw.get("AUTO_PUBLISH", "false") == "true"
|
|
1456
|
+
soak_mins = int(raw.get("SOAK_MINUTES", 30))
|
|
1457
|
+
|
|
1458
|
+
if _time.time() > soak_until:
|
|
1459
|
+
# Already expired (e.g. second restart after revert) — clean up and bail
|
|
1460
|
+
marker.unlink(missing_ok=True)
|
|
1461
|
+
return
|
|
1462
|
+
|
|
1463
|
+
cfg = cfg_loader.sentinel
|
|
1464
|
+
log_path = Path(".") / "logs" / "sentinel.log"
|
|
1465
|
+
patched_at_iso = raw.get("PATCHED_AT", _dt.now(_tz.utc).isoformat())
|
|
1466
|
+
|
|
1467
|
+
logger.info("Patch soak monitor started — hash=%s soak=%dm auto_publish=%s",
|
|
1468
|
+
patch_hash[:8], soak_mins, auto_publish)
|
|
1469
|
+
_slack_alert(cfg.slack_bot_token, cfg.slack_channel,
|
|
1470
|
+
f":hourglass: *Patch soak started* — monitoring for {soak_mins} min "
|
|
1471
|
+
f"(patch `{patch_hash[:8]}`). Will notify when done.")
|
|
1472
|
+
|
|
1473
|
+
check_interval = 60 # seconds
|
|
1474
|
+
error_found = False
|
|
1475
|
+
|
|
1476
|
+
while _time.time() < soak_until:
|
|
1477
|
+
await asyncio.sleep(check_interval)
|
|
1478
|
+
|
|
1479
|
+
# Scan sentinel.log for new ERROR lines since the patch
|
|
1480
|
+
if log_path.exists():
|
|
1481
|
+
try:
|
|
1482
|
+
text = log_path.read_text(encoding="utf-8", errors="replace")
|
|
1483
|
+
# Look for ERROR lines after our patch timestamp marker
|
|
1484
|
+
new_errors = [
|
|
1485
|
+
l for l in text.splitlines()
|
|
1486
|
+
if " ERROR " in l and l > patched_at_iso[:19]
|
|
1487
|
+
]
|
|
1488
|
+
if new_errors:
|
|
1489
|
+
error_found = True
|
|
1490
|
+
logger.error("Patch soak: new errors detected — reverting patch %s", patch_hash[:8])
|
|
1491
|
+
break
|
|
1492
|
+
except OSError:
|
|
1493
|
+
pass
|
|
1494
|
+
|
|
1495
|
+
marker.unlink(missing_ok=True)
|
|
1496
|
+
|
|
1497
|
+
if error_found:
|
|
1498
|
+
# Revert and restart
|
|
1499
|
+
loop = asyncio.get_event_loop()
|
|
1500
|
+
revert = await loop.run_in_executor(
|
|
1501
|
+
None,
|
|
1502
|
+
lambda: _sp.run(
|
|
1503
|
+
["git", "revert", "--no-edit", "HEAD"],
|
|
1504
|
+
cwd=str(code_dir), capture_output=True, text=True, timeout=60,
|
|
1505
|
+
),
|
|
1506
|
+
)
|
|
1507
|
+
revert_ok = revert.returncode == 0
|
|
1508
|
+
_slack_alert(cfg.slack_bot_token, cfg.slack_channel,
|
|
1509
|
+
f":x: *Patch soak failed* — new errors detected after patch `{patch_hash[:8]}`. "
|
|
1510
|
+
f"Reverted {'✓' if revert_ok else '(failed — check manually)'}. Restarting...")
|
|
1511
|
+
if revert_ok:
|
|
1512
|
+
import os as _os, sys as _sys
|
|
1513
|
+
await asyncio.sleep(2)
|
|
1514
|
+
_os.execv(_sys.executable, [_sys.executable] + _sys.argv)
|
|
1515
|
+
return
|
|
1516
|
+
|
|
1517
|
+
# Clean soak — notify
|
|
1518
|
+
remaining = max(0, int(soak_until - _time.time()))
|
|
1519
|
+
if auto_publish:
|
|
1520
|
+
# Sync patched Python into the npm package dir and publish
|
|
1521
|
+
import shutil as _sh
|
|
1522
|
+
npm_root = _sp.run(
|
|
1523
|
+
["npm", "root", "-g"], capture_output=True, text=True, timeout=30,
|
|
1524
|
+
).stdout.strip()
|
|
1525
|
+
pkg_dir = Path(npm_root) / "@misterhuydo" / "sentinel"
|
|
1526
|
+
if pkg_dir.exists():
|
|
1527
|
+
try:
|
|
1528
|
+
_sh.copytree(str(code_dir / "sentinel"), str(pkg_dir / "python" / "sentinel"),
|
|
1529
|
+
dirs_exist_ok=True)
|
|
1530
|
+
_sh.copytree(str(code_dir / "tests"), str(pkg_dir / "python" / "tests"),
|
|
1531
|
+
dirs_exist_ok=True)
|
|
1532
|
+
ver_result = _sp.run(
|
|
1533
|
+
["npm", "version", "patch", "--no-git-tag-version"],
|
|
1534
|
+
cwd=str(pkg_dir), capture_output=True, text=True, timeout=30,
|
|
1535
|
+
)
|
|
1536
|
+
new_ver = ver_result.stdout.strip()
|
|
1537
|
+
pub = _sp.run(
|
|
1538
|
+
["npm", "publish", "--access", "public"],
|
|
1539
|
+
cwd=str(pkg_dir), capture_output=True, text=True, timeout=120,
|
|
1540
|
+
)
|
|
1541
|
+
if pub.returncode == 0:
|
|
1542
|
+
_slack_alert(cfg.slack_bot_token, cfg.slack_channel,
|
|
1543
|
+
f":rocket: *Patch published* — `{new_ver}` is live on npm. "
|
|
1544
|
+
f"Users with `AUTO_UPGRADE=true` will receive it within "
|
|
1545
|
+
f"{cfg.upgrade_check_hours}h.")
|
|
1546
|
+
else:
|
|
1547
|
+
_slack_alert(cfg.slack_bot_token, cfg.slack_channel,
|
|
1548
|
+
f":white_check_mark: Soak clean — but `npm publish` failed: "
|
|
1549
|
+
f"```{pub.stderr[-300:]}```\nPublish manually or say *publish* to Boss.")
|
|
1550
|
+
except Exception as e:
|
|
1551
|
+
_slack_alert(cfg.slack_bot_token, cfg.slack_channel,
|
|
1552
|
+
f":white_check_mark: Soak clean — publish error: {e}. Publish manually.")
|
|
1553
|
+
else:
|
|
1554
|
+
_slack_alert(cfg.slack_bot_token, cfg.slack_channel,
|
|
1555
|
+
f":white_check_mark: *Patch soak clean* (`{patch_hash[:8]}`, {soak_mins} min). "
|
|
1556
|
+
f"npm package dir not found — publish manually.")
|
|
1557
|
+
else:
|
|
1558
|
+
_slack_alert(cfg.slack_bot_token, cfg.slack_channel,
|
|
1559
|
+
f":white_check_mark: *Patch soak clean* — `{patch_hash[:8]}` stable for {soak_mins} min. "
|
|
1560
|
+
f"Say *publish* when ready to release to all users.")
|
|
1561
|
+
|
|
1562
|
+
logger.info("Patch soak complete — hash=%s clean=True auto_publish=%s", patch_hash[:8], auto_publish)
|
|
1563
|
+
|
|
1564
|
+
|
|
1227
1565
|
async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
|
|
1228
1566
|
interval = cfg_loader.sentinel.poll_interval_seconds
|
|
1229
1567
|
logger.info("Sentinel starting — poll interval: %ds, repos: %s",
|
|
@@ -1249,6 +1587,12 @@ async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
|
|
|
1249
1587
|
else:
|
|
1250
1588
|
logger.info("Startup checks passed — startup email in 5 minutes")
|
|
1251
1589
|
|
|
1590
|
+
# Re-run Maven check on every config reload (SIGHUP or Boss reload_config)
|
|
1591
|
+
def _maven_reload_cb(cl):
|
|
1592
|
+
_check_maven_settings(cl, {})
|
|
1593
|
+
|
|
1594
|
+
cfg_loader.register_on_reload(_maven_reload_cb)
|
|
1595
|
+
|
|
1252
1596
|
asyncio.ensure_future(_send_startup_email_delayed(cfg_loader.sentinel, results))
|
|
1253
1597
|
asyncio.ensure_future(_config_poll_loop(cfg_loader))
|
|
1254
1598
|
if cfg_loader.sentinel.auto_upgrade:
|
|
@@ -1260,6 +1604,7 @@ async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
|
|
|
1260
1604
|
asyncio.ensure_future(run_slack_bot(cfg_loader, store))
|
|
1261
1605
|
if cfg_loader.sentinel.sentinel_dev_repo_path:
|
|
1262
1606
|
asyncio.ensure_future(_dev_poll_loop(cfg_loader, store))
|
|
1607
|
+
asyncio.ensure_future(_patch_soak_monitor(cfg_loader))
|
|
1263
1608
|
asyncio.ensure_future(_repo_task_poll_loop(cfg_loader, store))
|
|
1264
1609
|
|
|
1265
1610
|
while True:
|