@misterhuydo/sentinel 1.5.4 → 1.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/.cairn/.hint-lock +1 -1
  2. package/.cairn/minify-map.json +13 -1
  3. package/.cairn/session.json +2 -2
  4. package/.cairn/views/23edf4_sentinel_boss.py +3664 -0
  5. package/.cairn/views/5f5141_main.py +1067 -0
  6. package/.cairn/views/62a614_bundle.js +4 -1
  7. package/.cairn/views/7802b9_cicd_trigger.py +171 -0
  8. package/.cairn/views/ac3df4_repo_task_engine.py +351 -0
  9. package/lib/.cairn/minify-map.json +6 -0
  10. package/lib/.cairn/views/2a85cc_init.js +380 -0
  11. package/lib/.cairn/views/e26996_slack-setup.js +97 -0
  12. package/lib/.cairn/views/fb78ac_upgrade.js +36 -1
  13. package/lib/.cairn/views/fc4a1a_add.js +164 -51
  14. package/lib/init.js +54 -0
  15. package/lib/maven.js +212 -0
  16. package/lib/slack-setup.js +5 -0
  17. package/package.json +1 -1
  18. package/python/requirements.txt +1 -0
  19. package/python/sentinel/.cairn/.cairn-project +0 -0
  20. package/python/sentinel/.cairn/.hint-lock +1 -0
  21. package/python/sentinel/.cairn/session.json +9 -0
  22. package/python/sentinel/__pycache__/sentinel_boss.cpython-311.pyc +0 -0
  23. package/python/sentinel/config_loader.py +29 -10
  24. package/python/sentinel/dependency_manager.py +9 -2
  25. package/python/sentinel/git_manager.py +23 -0
  26. package/python/sentinel/issue_watcher.py +7 -1
  27. package/python/sentinel/main.py +353 -8
  28. package/python/sentinel/notify.py +44 -12
  29. package/python/sentinel/repo_task_engine.py +49 -7
  30. package/python/sentinel/sentinel_boss.py +117 -3
  31. package/python/sentinel/slack_bot.py +15 -2
  32. package/python/sentinel/state_store.py +0 -1
  33. package/python/tests/__init__.py +0 -0
  34. package/python/tests/test_config_loader.py +138 -0
  35. package/python/tests/test_log_parser.py +62 -0
  36. package/python/tests/test_repo_router.py +73 -0
  37. package/python/tests/test_smoke.py +96 -0
  38. package/python/tests/test_state_store.py +128 -0
@@ -78,6 +78,8 @@ class SentinelConfig:
78
78
  sync_max_file_mb: int = 200 # truncate synced log files exceeding this size (MB)
79
79
  boss_mode: str = "standard" # standard | strict | fun
80
80
  sentinel_dev_repo_path: str = "" # path to Sentinel source repo for Dev Claude
81
+ sentinel_dev_soak_minutes: int = 30 # minutes to monitor after a Patch restart before publishing
82
+ sentinel_dev_auto_publish: bool = False # if True, auto-publish to npm after clean soak
81
83
 
82
84
 
83
85
  @dataclass
@@ -112,8 +114,13 @@ class RepoConfig:
112
114
  cicd_user: str = "" # Jenkins username for Basic auth (defaults to "sentinel")
113
115
  health_url: str = "" # optional: HTTP endpoint returning {"Status": "true"}
114
116
  cicd_token: str = ""
115
- git_access: str = "" # ssh_user_key | ssh_deploy_key | https_token | https_pat
116
- ssh_key_file: str = "" # path to SSH private key; sets GIT_SSH_COMMAND when present
117
+ git_ssh_user_key: str = "" # personal GitHub SSH key (contributor on org repos)
118
+ git_ssh_deploy_key: str = "" # repo-specific deploy key (repos you own/admin)
119
+
120
+ @property
121
+ def ssh_key_file(self) -> str:
122
+ """Resolved SSH key — deploy key takes priority over user key."""
123
+ return self.git_ssh_deploy_key or self.git_ssh_user_key
117
124
 
118
125
 
119
126
  # ── Loader ────────────────────────────────────────────────────────────────────
@@ -123,6 +130,7 @@ class ConfigLoader:
123
130
  self.config_dir = Path(config_dir)
124
131
  self.sentinel: SentinelConfig = SentinelConfig()
125
132
  self.log_sources: dict[str, LogSourceConfig] = {}
133
+ self._on_reload_callbacks: list = [] # called after every load()
126
134
  self.repos: dict[str, RepoConfig] = {}
127
135
  self.load()
128
136
  self._register_sighup()
@@ -135,6 +143,15 @@ class ConfigLoader:
135
143
  "Config loaded: %d log-config(s), %d repo-config(s)",
136
144
  len(self.log_sources), len(self.repos),
137
145
  )
146
+ for cb in self._on_reload_callbacks:
147
+ try:
148
+ cb(self)
149
+ except Exception as e:
150
+ logger.warning("Config reload callback failed: %s", e)
151
+
152
+ def register_on_reload(self, callback) -> None:
153
+ """Register a callback(cfg_loader) to be called after every config reload."""
154
+ self._on_reload_callbacks.append(callback)
138
155
 
139
156
  def _load_sentinel(self):
140
157
  # Load workspace-level config first (~/sentinel/sentinel.properties),
@@ -218,6 +235,8 @@ class ConfigLoader:
218
235
  raw_mode = d.get("BOSS_MODE", "standard").lower().strip()
219
236
  c.boss_mode = raw_mode if raw_mode in ("standard", "strict", "fun") else "standard"
220
237
  c.sentinel_dev_repo_path = d.get("SENTINEL_DEV_REPO_PATH", "")
238
+ c.sentinel_dev_soak_minutes = int(d.get("SENTINEL_DEV_SOAK_MINUTES", 30))
239
+ c.sentinel_dev_auto_publish = d.get("SENTINEL_DEV_AUTO_PUBLISH", "false").lower() == "true"
221
240
  self.sentinel = c
222
241
 
223
242
  def _load_log_sources(self):
@@ -254,8 +273,8 @@ class ConfigLoader:
254
273
  # Project-level defaults from sentinel.properties
255
274
  sentinel_path = self.config_dir / "sentinel.properties"
256
275
  proj_d: dict[str, str] = _parse_properties(str(sentinel_path)) if sentinel_path.exists() else {}
257
- default_git_access = proj_d.get("GIT_ACCESS", "")
258
- default_git_ssh_key = os.path.expanduser(proj_d.get("GIT_SSH_KEY", ""))
276
+ default_user_key = os.path.expanduser(proj_d.get("GIT_SSH_USER_KEY", ""))
277
+ default_deploy_key = os.path.expanduser(proj_d.get("GIT_SSH_DEPLOY_KEY", ""))
259
278
 
260
279
  self.repos = {}
261
280
  for path in sorted(repos_dir.glob("*.properties")):
@@ -273,15 +292,15 @@ class ConfigLoader:
273
292
  r.cicd_user = d.get("CICD_USER", "")
274
293
  r.cicd_token = d.get("CICD_TOKEN", "")
275
294
  r.health_url = d.get("HEALTH_URL", "")
276
- r.git_access = d.get("GIT_ACCESS", default_git_access)
277
- # GIT_SSH_KEY (preferred) or legacy SSH_KEY_FILE, then project default
278
- raw_key = d.get("GIT_SSH_KEY", "") or d.get("SSH_KEY_FILE", "")
279
- r.ssh_key_file = os.path.expanduser(raw_key) if raw_key else default_git_ssh_key
295
+ raw_user_key = d.get("GIT_SSH_USER_KEY", "")
296
+ raw_deploy_key = d.get("GIT_SSH_DEPLOY_KEY", "") or d.get("GIT_SSH_KEY", "") or d.get("SSH_KEY_FILE", "")
297
+ r.git_ssh_user_key = os.path.expanduser(raw_user_key) if raw_user_key else default_user_key
298
+ r.git_ssh_deploy_key = os.path.expanduser(raw_deploy_key) if raw_deploy_key else default_deploy_key
280
299
  # Auto-discover deploy key in project dir if no key configured
281
- if not r.ssh_key_file:
300
+ if not r.git_ssh_deploy_key and not r.git_ssh_user_key:
282
301
  auto_key = Path(self.config_dir).parent / f"{r.repo_name}.key"
283
302
  if auto_key.exists():
284
- r.ssh_key_file = str(auto_key.resolve())
303
+ r.git_ssh_deploy_key = str(auto_key.resolve())
285
304
  self.repos[r.repo_name] = r
286
305
 
287
306
  def _register_sighup(self):
@@ -232,8 +232,8 @@ def execute_cascade(
232
232
  Returns one CascadeResult per dependent repo attempted.
233
233
  """
234
234
  # Import here to avoid circular imports
235
- from .git_manager import commit_file_change, push_dep_update, open_dep_pr, _git_env, _git, maven_compile_check, MissingToolError # noqa: E501
236
- from .notify import notify_cascade_build_failed
235
+ from .git_manager import commit_file_change, push_dep_update, open_dep_pr, _git_env, _git, maven_compile_check, MissingToolError, MavenAuthError # noqa: E501
236
+ from .notify import notify_cascade_build_failed, notify_nexus_auth_failure
237
237
 
238
238
  all_dependents = find_dependents(artifact_id, repos, skip_repo=source_repo_name)
239
239
  if target_repo_names:
@@ -264,6 +264,13 @@ def execute_cascade(
264
264
  # Dry-run: compile before committing — catch broken deps early
265
265
  try:
266
266
  compile_ok, compile_output = maven_compile_check(repo.local_path)
267
+ except MavenAuthError as e:
268
+ _git(["checkout", "pom.xml"], cwd=repo.local_path, env=_git_env(repo))
269
+ result.error = "Maven auth failed — Nexus credentials missing or invalid"
270
+ logger.error("Cascade auth failure for %s: %s", repo.repo_name, e.output[-300:])
271
+ notify_nexus_auth_failure(cfg, repo.repo_name, f"cascade dep update {artifact_id}→{new_version}", e.output)
272
+ results.append(result)
273
+ continue
267
274
  except MissingToolError:
268
275
  compile_ok, compile_output = False, "mvn not installed on this server"
269
276
 
@@ -29,6 +29,25 @@ class MissingToolError(Exception):
29
29
  super().__init__(f"Build tool '{tool}' not found on this server")
30
30
  self.tool = tool
31
31
 
32
+
33
+ class MavenAuthError(Exception):
34
+ """Raised when mvn fails due to Nexus/Artifactory credential issues (401/403)."""
35
+ def __init__(self, output: str):
36
+ super().__init__("Maven dependency resolution failed — Nexus credentials missing or invalid")
37
+ self.output = output
38
+
39
+
40
+ _NEXUS_AUTH_RE = re.compile(
41
+ r"(DependencyResolutionException"
42
+ r"|Could not transfer artifact"
43
+ r"|RepoAuthenticationException"
44
+ r"|status code: 40[13]"
45
+ r"|401 Unauthorized"
46
+ r"|403 Forbidden"
47
+ r"|authentication failed)",
48
+ re.IGNORECASE,
49
+ )
50
+
32
51
  # Files that must never be modified by Sentinel
33
52
  _PROTECTED_PATHS = {".github/", "Jenkinsfile"}
34
53
 
@@ -69,6 +88,8 @@ def maven_compile_check(local_path: str, timeout: int = 300) -> tuple[bool, str]
69
88
  timeout=timeout,
70
89
  )
71
90
  output = (r.stdout + r.stderr).strip()
91
+ if r.returncode != 0 and _NEXUS_AUTH_RE.search(output):
92
+ raise MavenAuthError(output)
72
93
  return r.returncode == 0, output
73
94
 
74
95
 
@@ -160,6 +181,8 @@ def _run_tests(repo: RepoConfig, local_path: str) -> bool:
160
181
  if r.returncode != 0:
161
182
  output = r.stdout[-3000:] + r.stderr[-1000:]
162
183
  logger.error("Tests failed:\n%s", output)
184
+ if _NEXUS_AUTH_RE.search(output):
185
+ raise MavenAuthError(output)
163
186
  # Detect missing sub-tool invoked by the build (e.g. yarn called from Maven exec plugin)
164
187
  import re as _re
165
188
  m = _re.search(r'Cannot run program "([^"]+)"', output)
@@ -42,6 +42,7 @@ class IssueEvent:
42
42
  severity: str = "ERROR"
43
43
  timestamp: str = ""
44
44
  submitter_user_id: str = "" # Slack user ID who raised this via Boss, if known
45
+ origin_channel: str = "" # Slack channel where the issue was raised, if known
45
46
 
46
47
  # Compatibility fields matching ErrorEvent interface
47
48
  level: str = "ERROR"
@@ -130,8 +131,9 @@ def scan_issues(project_dir: Path) -> list[IssueEvent]:
130
131
 
131
132
  # Parse metadata headers in any order (TARGET_REPO, SUBMITTED_BY, SUBMITTED_AT, etc.)
132
133
  import re as _re
133
- _META = ("TARGET_REPO:", "SUBMITTED_BY:", "SUBMITTED_AT:", "SUPPORT_URL:")
134
+ _META = ("TARGET_REPO:", "SUBMITTED_BY:", "SUBMITTED_AT:", "SUPPORT_URL:", "ORIGIN_CHANNEL:")
134
135
  submitter_user_id = ""
136
+ origin_channel = ""
135
137
  for i, line in enumerate(lines):
136
138
  stripped = line.strip()
137
139
  upper = stripped.upper()
@@ -143,6 +145,9 @@ def scan_issues(project_dir: Path) -> list[IssueEvent]:
143
145
  if m:
144
146
  submitter_user_id = m.group(1)
145
147
  body_start = i + 1
148
+ elif upper.startswith("ORIGIN_CHANNEL:"):
149
+ origin_channel = stripped[len("ORIGIN_CHANNEL:"):].strip()
150
+ body_start = i + 1
146
151
  elif any(upper.startswith(p) for p in _META) or not stripped:
147
152
  body_start = i + 1
148
153
  else:
@@ -158,6 +163,7 @@ def scan_issues(project_dir: Path) -> list[IssueEvent]:
158
163
  body=body,
159
164
  target_repo=target_repo,
160
165
  submitter_user_id=submitter_user_id,
166
+ origin_channel=origin_channel,
161
167
  ))
162
168
  logger.info("Found issue: %s (target_repo=%r)", f.name, target_repo or "auto")
163
169
 
@@ -23,7 +23,7 @@ from pathlib import Path
23
23
  from .cairn_client import ensure_installed as cairn_installed, index_repo
24
24
  from .config_loader import ConfigLoader, SentinelConfig
25
25
  from .fix_engine import generate_fix
26
- from .git_manager import apply_and_commit, publish, _git_env, MissingToolError, poll_open_prs
26
+ from .git_manager import apply_and_commit, publish, _git_env, MissingToolError, MavenAuthError, poll_open_prs
27
27
  from .cicd_trigger import trigger as cicd_trigger
28
28
  from .log_fetcher import fetch_all
29
29
  from .log_parser import parse_all, scan_all_for_markers, ErrorEvent
@@ -223,6 +223,12 @@ async def _handle_error(event: ErrorEvent, cfg_loader: ConfigLoader, store: Stat
223
223
 
224
224
  try:
225
225
  commit_status, commit_hash = apply_and_commit(event, patch_path, repo, sentinel)
226
+ except MavenAuthError as e:
227
+ from .notify import notify_nexus_auth_failure
228
+ logger.error("Nexus auth failure applying fix for %s: %s", event.fingerprint, str(e))
229
+ notify_nexus_auth_failure(sentinel, repo.repo_name, f"fix {event.fingerprint[:8]}", e.output)
230
+ store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
231
+ return
226
232
  except MissingToolError as e:
227
233
  logger.warning("Missing tool for %s: %s", event.source, e)
228
234
  if _auto_install_if_safe(e.tool, repo.local_path, sentinel, repo.repo_name, event.source):
@@ -319,13 +325,14 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
319
325
  # Post "working on" to channel and capture thread_ts for progress replies
320
326
  from .notify import slack_alert as _slack_alert, slack_thread_reply as _slack_reply
321
327
  _submitter = getattr(event, "submitter_user_id", "")
328
+ _origin_channel = getattr(event, "origin_channel", "") or sentinel.slack_channel
322
329
  _started_msg = (
323
330
  f":hammer: Working on *<@{_submitter}>*'s request — *{repo.repo_name}*\n"
324
331
  f"_{event.message[:120]}_"
325
332
  ) if _submitter else (
326
333
  f":hammer: Working on *{repo.repo_name}*\n_{event.message[:120]}_"
327
334
  )
328
- _thread_ts = _slack_alert(sentinel.slack_bot_token, sentinel.slack_channel, _started_msg)
335
+ _thread_ts = _slack_alert(sentinel.slack_bot_token, _origin_channel, _started_msg)
329
336
 
330
337
  def _progress(msg: str) -> None:
331
338
  """Post a threaded reply under the 'Working on' message."""
@@ -402,7 +409,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
402
409
  _progress(f":arrow_right: <{pr_url}|PR opened> — awaiting review")
403
410
  notify_fix_applied(sentinel, event.source, event.message,
404
411
  repo_name=repo.repo_name, branch=branch, pr_url=pr_url,
405
- submitter_user_id=submitter_uid)
412
+ submitter_user_id=submitter_uid, origin_channel=_origin_channel)
406
413
  mark_done(event.issue_file)
407
414
 
408
415
  if repo.auto_publish:
@@ -415,6 +422,16 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
415
422
  return {"submitter": submitter_uid, "repo_name": repo.repo_name,
416
423
  "status": "done", "summary": event.message[:120], "pr_url": pr_url}
417
424
 
425
+ except MavenAuthError as e:
426
+ from .notify import notify_nexus_auth_failure
427
+ submitter_uid = getattr(event, "submitter_user_id", "")
428
+ logger.error("Nexus auth failure for issue %s: %s", event.fingerprint, str(e))
429
+ _progress(":lock: Maven authentication failed — DMing admin for Nexus credentials")
430
+ notify_nexus_auth_failure(sentinel, repo.repo_name, f"issue fix {event.fingerprint[:8]}", e.output)
431
+ store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
432
+ mark_done(event.issue_file)
433
+ return {"submitter": submitter_uid, "repo_name": repo.repo_name,
434
+ "status": "blocked", "summary": "Nexus credentials needed", "pr_url": ""}
418
435
  except MissingToolError as e:
419
436
  logger.warning("Missing tool for %s: %s", event.source, e)
420
437
  submitter_uid = getattr(event, "submitter_user_id", "")
@@ -424,7 +441,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
424
441
  )
425
442
  if not installed:
426
443
  _progress(f":x: `{e.tool}` is not a known safe tool — manual install required")
427
- notify_missing_tool(sentinel, e.tool, repo.repo_name, event.source, submitter_uid)
444
+ notify_missing_tool(sentinel, e.tool, repo.repo_name, event.source, submitter_uid, _origin_channel)
428
445
  store.record_fix(event.fingerprint, "failed", repo_name=repo.repo_name)
429
446
  mark_done(event.issue_file)
430
447
  return {"submitter": submitter_uid, "repo_name": repo.repo_name,
@@ -448,7 +465,8 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
448
465
  _progress(":x: Tests still failing after tool install — needs human review")
449
466
  notify_fix_blocked(sentinel, event.source, event.message,
450
467
  reason="Patch was generated but commit/tests failed after tool install",
451
- repo_name=repo.repo_name, submitter_user_id=submitter_uid)
468
+ repo_name=repo.repo_name, submitter_user_id=submitter_uid,
469
+ origin_channel=_origin_channel)
452
470
  mark_done(event.issue_file)
453
471
  return {"submitter": submitter_uid, "repo_name": repo.repo_name,
454
472
  "status": "blocked", "summary": "Commit/tests failed after tool install", "pr_url": ""}
@@ -472,7 +490,7 @@ async def _handle_issue(event: IssueEvent, cfg_loader: ConfigLoader, store: Stat
472
490
  _progress(f":arrow_right: <{pr_url}|PR opened> — awaiting review")
473
491
  notify_fix_applied(sentinel, event.source, event.message,
474
492
  repo_name=repo.repo_name, branch=branch, pr_url=pr_url,
475
- submitter_user_id=submitter_uid)
493
+ submitter_user_id=submitter_uid, origin_channel=_origin_channel)
476
494
  mark_done(event.issue_file)
477
495
  if repo.auto_publish:
478
496
  ok = cicd_trigger(repo, store, event.fingerprint)
@@ -721,9 +739,125 @@ async def _startup_checks(cfg_loader: ConfigLoader) -> dict:
721
739
  results["ssh"].append({"name": src_name, "host": host,
722
740
  "status": "error", "message": str(e)})
723
741
 
742
+ # ── Maven settings.xml check ─────────────────────────────────────────────
743
+ _check_maven_settings(cfg_loader, results)
744
+
724
745
  return results
725
746
 
726
747
 
748
+ def _check_maven_settings(cfg_loader, results: dict) -> None:
749
+ """
750
+ Scan pom.xml files in all cloned repos for private Nexus <repository> entries.
751
+ Groups missing credentials by hostname and DMs the first configured admin user
752
+ with two options: paste full settings.xml content, or provide creds per host.
753
+ """
754
+ import re as _re
755
+ import os as _os
756
+ from pathlib import Path as _Path
757
+ from urllib.parse import urlparse as _urlparse
758
+
759
+ settings_path = _Path(_os.path.expanduser("~/.m2/settings.xml"))
760
+
761
+ _PUBLIC = _re.compile(r"repo1\.maven\.org|central\.maven\.org|repo\.maven\.apache\.org", _re.I)
762
+ _REPO_RE = _re.compile(
763
+ r"<(?:repository|snapshotRepository|pluginRepository)>"
764
+ r"\s*<id>([^<]+)</id>\s*<url>([^<]+)</url>",
765
+ _re.S,
766
+ )
767
+ _SERVER_RE = _re.compile(r"<server>\s*<id>([^<]+)</id>", _re.S)
768
+
769
+ # Scan all cloned repos — group by hostname → set of repo IDs
770
+ nexus_map: dict[str, set[str]] = {}
771
+ for repo in cfg_loader.repos.values():
772
+ local = _Path(repo.local_path)
773
+ if not local.exists():
774
+ continue
775
+ for pom in local.rglob("pom.xml"):
776
+ if any(p in ("target", ".git", "node_modules") for p in pom.parts):
777
+ continue
778
+ try:
779
+ text = pom.read_text(encoding="utf-8", errors="replace")
780
+ except OSError:
781
+ continue
782
+ for m in _REPO_RE.finditer(text):
783
+ repo_id = m.group(1).strip()
784
+ url = m.group(2).strip()
785
+ if _PUBLIC.search(url):
786
+ continue
787
+ try:
788
+ hostname = _urlparse(url).hostname or ""
789
+ except Exception:
790
+ continue
791
+ if hostname:
792
+ nexus_map.setdefault(hostname, set()).add(repo_id)
793
+
794
+ if not nexus_map:
795
+ return
796
+
797
+ # Read configured server IDs from existing settings.xml
798
+ configured_ids: set[str] = set()
799
+ if settings_path.exists():
800
+ try:
801
+ for m in _SERVER_RE.finditer(settings_path.read_text(encoding="utf-8", errors="replace")):
802
+ configured_ids.add(m.group(1).strip())
803
+ except OSError:
804
+ pass
805
+
806
+ # Filter to hosts with at least one unconfigured ID
807
+ missing_hosts: dict[str, set[str]] = {
808
+ host: ids - configured_ids
809
+ for host, ids in nexus_map.items()
810
+ if ids - configured_ids
811
+ }
812
+
813
+ if not missing_hosts:
814
+ all_ids = {rid for ids in nexus_map.values() for rid in ids}
815
+ logger.info("Maven settings OK — %d repo ID(s) configured", len(all_ids))
816
+ return
817
+
818
+ summary = "; ".join(
819
+ f"{host} ({', '.join(sorted(ids))})"
820
+ for host, ids in sorted(missing_hosts.items())
821
+ )
822
+ results["warnings"].append(f"Maven credentials missing: {summary}")
823
+ logger.warning("Maven settings gap: %s", summary)
824
+
825
+ # DM the first admin user with two resolution options
826
+ cfg = cfg_loader.sentinel
827
+ if not cfg.slack_admin_users or not cfg.slack_bot_token:
828
+ logger.warning(
829
+ "Cannot DM admin about Maven credentials — "
830
+ "set SLACK_ADMIN_USERS and SLACK_BOT_TOKEN in sentinel.properties"
831
+ )
832
+ return
833
+
834
+ lines = [
835
+ ":warning: *Maven credentials needed* — I found Nexus servers missing from `~/.m2/settings.xml`:",
836
+ ]
837
+ for host, ids in sorted(missing_hosts.items()):
838
+ lines.append(f" • `{host}` → repo IDs: `{', '.join(sorted(ids))}`")
839
+ lines += [
840
+ "",
841
+ "Pick one of these two ways to fix it:",
842
+ "",
843
+ "*Option 1 — paste your existing `settings.xml`*",
844
+ "Copy the full content of your `~/.m2/settings.xml` and send it to me starting with:",
845
+ "`nexus settings`",
846
+ "then the XML on the next line(s). I'll extract the credentials and save them.",
847
+ "",
848
+ "*Option 2 — provide credentials per server*",
849
+ "Send one message per Nexus host:",
850
+ "`nexus creds <host> <username> <password>`",
851
+ "",
852
+ "Examples:",
853
+ ]
854
+ for host in sorted(missing_hosts):
855
+ lines.append(f"`nexus creds {host} myuser mypassword`")
856
+
857
+ from .notify import slack_dm as _slack_dm
858
+ _slack_dm(cfg.slack_bot_token, cfg.slack_admin_users[0], "\n".join(lines))
859
+
860
+
727
861
  async def _send_startup_email_delayed(cfg, results: dict, delay: int = 300):
728
862
  """Wait delay seconds then send startup summary email."""
729
863
  await asyncio.sleep(delay)
@@ -1010,8 +1144,73 @@ async def _handle_dev_task(task, cfg_loader: ConfigLoader, store: StateStore):
1010
1144
  if status == "done":
1011
1145
  _slack_alert(
1012
1146
  sentinel.slack_bot_token, sentinel.slack_channel,
1013
- f"{mentions}:white_check_mark: *Patch finished* — changes committed to Sentinel source.",
1147
+ f"{mentions}:white_check_mark: *Patch finished* — running tests before restart...",
1014
1148
  )
1149
+ # Run test suite before restarting — revert if tests fail.
1150
+ import os as _os, sys as _sys, subprocess as _sp
1151
+ _loop2 = asyncio.get_event_loop()
1152
+ _code_dir = Path(sentinel.sentinel_dev_repo_path or ".")
1153
+ _venv_pytest = _code_dir / ".venv" / "bin" / "pytest"
1154
+ _pytest_bin = str(_venv_pytest) if _venv_pytest.exists() else "pytest"
1155
+ try:
1156
+ _test_result = await _loop2.run_in_executor(
1157
+ None,
1158
+ lambda: _sp.run(
1159
+ [_pytest_bin, "tests/", "-q", "--tb=short", "--no-header"],
1160
+ cwd=str(_code_dir), capture_output=True, text=True, timeout=120,
1161
+ ),
1162
+ )
1163
+ except Exception as _e:
1164
+ _slack_alert(
1165
+ sentinel.slack_bot_token, sentinel.slack_channel,
1166
+ f"{mentions}:warning: *Patch* — could not run tests: {_e}. Restarting anyway.",
1167
+ )
1168
+ _test_result = None
1169
+
1170
+ if _test_result is not None and _test_result.returncode != 0:
1171
+ # Tests failed — revert the fix and stay on old code
1172
+ _revert = await _loop2.run_in_executor(
1173
+ None,
1174
+ lambda: _sp.run(
1175
+ ["git", "revert", "--no-edit", "HEAD"],
1176
+ cwd=str(_code_dir), capture_output=True, text=True, timeout=60,
1177
+ ),
1178
+ )
1179
+ _revert_ok = _revert.returncode == 0
1180
+ _test_summary = (_test_result.stdout + _test_result.stderr)[-800:]
1181
+ _slack_alert(
1182
+ sentinel.slack_bot_token, sentinel.slack_channel,
1183
+ f"{mentions}:x: *Patch tests failed* — fix reverted{'✓' if _revert_ok else ' (revert failed — check manually)'}.\n```{_test_summary}```",
1184
+ )
1185
+ logger.error("Patch tests failed — reverted. Output:\n%s", _test_summary)
1186
+ else:
1187
+ _summary = ""
1188
+ if _test_result:
1189
+ _lines = _test_result.stdout.strip().splitlines()
1190
+ _summary = f" ({_lines[-1]})" if _lines else ""
1191
+ _slack_alert(
1192
+ sentinel.slack_bot_token, sentinel.slack_channel,
1193
+ f"{mentions}:white_check_mark: *Patch applied* — tests passed{_summary}. Restarting...",
1194
+ )
1195
+ logger.info("Patch tests passed — restarting Sentinel (execv)")
1196
+ # Write soak marker so the next boot knows to monitor for regressions.
1197
+ _patch_hash = _sp.run(
1198
+ ["git", "rev-parse", "HEAD"],
1199
+ cwd=str(_code_dir), capture_output=True, text=True,
1200
+ ).stdout.strip()
1201
+ from datetime import datetime as _dt, timezone as _tz
1202
+ _soak_mins = cfg_loader.sentinel.sentinel_dev_soak_minutes
1203
+ _soak_until = (_dt.now(_tz.utc).timestamp() + _soak_mins * 60)
1204
+ _auto_pub = cfg_loader.sentinel.sentinel_dev_auto_publish
1205
+ (_code_dir / ".patch-soak").write_text(
1206
+ f"PATCH_HASH={_patch_hash}\n"
1207
+ f"PATCHED_AT={_dt.now(_tz.utc).isoformat()}\n"
1208
+ f"SOAK_UNTIL={_soak_until}\n"
1209
+ f"AUTO_PUBLISH={str(_auto_pub).lower()}\n"
1210
+ f"SOAK_MINUTES={_soak_mins}\n"
1211
+ )
1212
+ await asyncio.sleep(1) # let the Slack message flush
1213
+ _os.execv(_sys.executable, [_sys.executable] + _sys.argv)
1015
1214
  elif status == "needs_human":
1016
1215
  # Boss qualifies the raw Patch explanation before surfacing to users
1017
1216
  qualified = _boss_qualify_dev_reason(detail, sentinel)
@@ -1128,7 +1327,13 @@ async def _handle_repo_task(task, repo_cfg, cfg_loader: ConfigLoader, store: Sta
1128
1327
  mentions = (mentions + " ") if mentions else ""
1129
1328
 
1130
1329
  if status == "done":
1131
- if detail: # PR URL
1330
+ if detail and detail.startswith("__cicd__"):
1331
+ cicd_name = detail[len("__cicd__"):]
1332
+ _slack_alert(
1333
+ sentinel.slack_bot_token, sentinel.slack_channel,
1334
+ f"{mentions}:white_check_mark: Done — pushed to `{task.repo_name}/{repo_cfg.branch}` and triggered `{cicd_name}` release.",
1335
+ )
1336
+ elif detail: # PR URL
1132
1337
  _slack_alert(
1133
1338
  sentinel.slack_bot_token, sentinel.slack_channel,
1134
1339
  f"{mentions}:white_check_mark: Done — PR opened for `{task.repo_name}`: {detail}",
@@ -1224,6 +1429,139 @@ def _log_auth_status(cfg: SentinelConfig) -> None:
1224
1429
  slack_alert(cfg.slack_bot_token, cfg.slack_channel, msg)
1225
1430
 
1226
1431
 
1432
+ async def _patch_soak_monitor(cfg_loader: ConfigLoader) -> None:
1433
+ """
1434
+ After a Patch-triggered restart, monitor sentinel.log for N minutes.
1435
+ - Clean soak → notify Slack; if AUTO_PUBLISH=true → npm publish
1436
+ - Crash detected → git revert HEAD, restart, alert Slack
1437
+ """
1438
+ import subprocess as _sp, time as _time
1439
+ from datetime import datetime as _dt, timezone as _tz
1440
+ from .notify import slack_alert as _slack_alert
1441
+
1442
+ code_dir = Path(cfg_loader.sentinel.sentinel_dev_repo_path)
1443
+ marker = code_dir / ".patch-soak"
1444
+ if not marker.exists():
1445
+ return
1446
+
1447
+ raw = {}
1448
+ for line in marker.read_text().splitlines():
1449
+ if "=" in line:
1450
+ k, _, v = line.partition("=")
1451
+ raw[k.strip()] = v.strip()
1452
+
1453
+ patch_hash = raw.get("PATCH_HASH", "unknown")
1454
+ soak_until = float(raw.get("SOAK_UNTIL", 0))
1455
+ auto_publish = raw.get("AUTO_PUBLISH", "false") == "true"
1456
+ soak_mins = int(raw.get("SOAK_MINUTES", 30))
1457
+
1458
+ if _time.time() > soak_until:
1459
+ # Already expired (e.g. second restart after revert) — clean up and bail
1460
+ marker.unlink(missing_ok=True)
1461
+ return
1462
+
1463
+ cfg = cfg_loader.sentinel
1464
+ log_path = Path(".") / "logs" / "sentinel.log"
1465
+ patched_at_iso = raw.get("PATCHED_AT", _dt.now(_tz.utc).isoformat())
1466
+
1467
+ logger.info("Patch soak monitor started — hash=%s soak=%dm auto_publish=%s",
1468
+ patch_hash[:8], soak_mins, auto_publish)
1469
+ _slack_alert(cfg.slack_bot_token, cfg.slack_channel,
1470
+ f":hourglass: *Patch soak started* — monitoring for {soak_mins} min "
1471
+ f"(patch `{patch_hash[:8]}`). Will notify when done.")
1472
+
1473
+ check_interval = 60 # seconds
1474
+ error_found = False
1475
+
1476
+ while _time.time() < soak_until:
1477
+ await asyncio.sleep(check_interval)
1478
+
1479
+ # Scan sentinel.log for new ERROR lines since the patch
1480
+ if log_path.exists():
1481
+ try:
1482
+ text = log_path.read_text(encoding="utf-8", errors="replace")
1483
+ # Look for ERROR lines after our patch timestamp marker
1484
+ new_errors = [
1485
+ l for l in text.splitlines()
1486
+ if " ERROR " in l and l > patched_at_iso[:19]
1487
+ ]
1488
+ if new_errors:
1489
+ error_found = True
1490
+ logger.error("Patch soak: new errors detected — reverting patch %s", patch_hash[:8])
1491
+ break
1492
+ except OSError:
1493
+ pass
1494
+
1495
+ marker.unlink(missing_ok=True)
1496
+
1497
+ if error_found:
1498
+ # Revert and restart
1499
+ loop = asyncio.get_event_loop()
1500
+ revert = await loop.run_in_executor(
1501
+ None,
1502
+ lambda: _sp.run(
1503
+ ["git", "revert", "--no-edit", "HEAD"],
1504
+ cwd=str(code_dir), capture_output=True, text=True, timeout=60,
1505
+ ),
1506
+ )
1507
+ revert_ok = revert.returncode == 0
1508
+ _slack_alert(cfg.slack_bot_token, cfg.slack_channel,
1509
+ f":x: *Patch soak failed* — new errors detected after patch `{patch_hash[:8]}`. "
1510
+ f"Reverted {'✓' if revert_ok else '(failed — check manually)'}. Restarting...")
1511
+ if revert_ok:
1512
+ import os as _os, sys as _sys
1513
+ await asyncio.sleep(2)
1514
+ _os.execv(_sys.executable, [_sys.executable] + _sys.argv)
1515
+ return
1516
+
1517
+ # Clean soak — notify
1518
+ remaining = max(0, int(soak_until - _time.time()))
1519
+ if auto_publish:
1520
+ # Sync patched Python into the npm package dir and publish
1521
+ import shutil as _sh
1522
+ npm_root = _sp.run(
1523
+ ["npm", "root", "-g"], capture_output=True, text=True, timeout=30,
1524
+ ).stdout.strip()
1525
+ pkg_dir = Path(npm_root) / "@misterhuydo" / "sentinel"
1526
+ if pkg_dir.exists():
1527
+ try:
1528
+ _sh.copytree(str(code_dir / "sentinel"), str(pkg_dir / "python" / "sentinel"),
1529
+ dirs_exist_ok=True)
1530
+ _sh.copytree(str(code_dir / "tests"), str(pkg_dir / "python" / "tests"),
1531
+ dirs_exist_ok=True)
1532
+ ver_result = _sp.run(
1533
+ ["npm", "version", "patch", "--no-git-tag-version"],
1534
+ cwd=str(pkg_dir), capture_output=True, text=True, timeout=30,
1535
+ )
1536
+ new_ver = ver_result.stdout.strip()
1537
+ pub = _sp.run(
1538
+ ["npm", "publish", "--access", "public"],
1539
+ cwd=str(pkg_dir), capture_output=True, text=True, timeout=120,
1540
+ )
1541
+ if pub.returncode == 0:
1542
+ _slack_alert(cfg.slack_bot_token, cfg.slack_channel,
1543
+ f":rocket: *Patch published* — `{new_ver}` is live on npm. "
1544
+ f"Users with `AUTO_UPGRADE=true` will receive it within "
1545
+ f"{cfg.upgrade_check_hours}h.")
1546
+ else:
1547
+ _slack_alert(cfg.slack_bot_token, cfg.slack_channel,
1548
+ f":white_check_mark: Soak clean — but `npm publish` failed: "
1549
+ f"```{pub.stderr[-300:]}```\nPublish manually or say *publish* to Boss.")
1550
+ except Exception as e:
1551
+ _slack_alert(cfg.slack_bot_token, cfg.slack_channel,
1552
+ f":white_check_mark: Soak clean — publish error: {e}. Publish manually.")
1553
+ else:
1554
+ _slack_alert(cfg.slack_bot_token, cfg.slack_channel,
1555
+ f":white_check_mark: *Patch soak clean* (`{patch_hash[:8]}`, {soak_mins} min). "
1556
+ f"npm package dir not found — publish manually.")
1557
+ else:
1558
+ _slack_alert(cfg.slack_bot_token, cfg.slack_channel,
1559
+ f":white_check_mark: *Patch soak clean* — `{patch_hash[:8]}` stable for {soak_mins} min. "
1560
+ f"Say *publish* when ready to release to all users.")
1561
+
1562
+ logger.info("Patch soak complete — hash=%s clean=True auto_publish=%s", patch_hash[:8], auto_publish)
1563
+
1564
+
1227
1565
  async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
1228
1566
  interval = cfg_loader.sentinel.poll_interval_seconds
1229
1567
  logger.info("Sentinel starting — poll interval: %ds, repos: %s",
@@ -1249,6 +1587,12 @@ async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
1249
1587
  else:
1250
1588
  logger.info("Startup checks passed — startup email in 5 minutes")
1251
1589
 
1590
+ # Re-run Maven check on every config reload (SIGHUP or Boss reload_config)
1591
+ def _maven_reload_cb(cl):
1592
+ _check_maven_settings(cl, {})
1593
+
1594
+ cfg_loader.register_on_reload(_maven_reload_cb)
1595
+
1252
1596
  asyncio.ensure_future(_send_startup_email_delayed(cfg_loader.sentinel, results))
1253
1597
  asyncio.ensure_future(_config_poll_loop(cfg_loader))
1254
1598
  if cfg_loader.sentinel.auto_upgrade:
@@ -1260,6 +1604,7 @@ async def run_loop(cfg_loader: ConfigLoader, store: StateStore):
1260
1604
  asyncio.ensure_future(run_slack_bot(cfg_loader, store))
1261
1605
  if cfg_loader.sentinel.sentinel_dev_repo_path:
1262
1606
  asyncio.ensure_future(_dev_poll_loop(cfg_loader, store))
1607
+ asyncio.ensure_future(_patch_soak_monitor(cfg_loader))
1263
1608
  asyncio.ensure_future(_repo_task_poll_loop(cfg_loader, store))
1264
1609
 
1265
1610
  while True: