loki-mode 11.128.1 → 11.129.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +1 -0
  2. package/SKILL.md +3 -3
  3. package/VERSION +1 -1
  4. package/autonomy/lib/help-core.sh +30 -42
  5. package/autonomy/lib/workspace.py +292 -19
  6. package/autonomy/lib/workspace_comment.py +40 -11
  7. package/autonomy/loki +109 -169
  8. package/autonomy/route.sh +8 -4
  9. package/autonomy/trigger-server.py +373 -32
  10. package/bin/loki +77 -14
  11. package/completions/_loki +11 -142
  12. package/completions/loki.bash +4 -2
  13. package/dashboard/__init__.py +1 -1
  14. package/docs/CLI-CORE-VERBS.md +61 -37
  15. package/docs/CLI-INVENTORY.md +152 -153
  16. package/docs/CLI-REFERENCE.md +20 -166
  17. package/docs/DEPLOYMENT.md +38 -3
  18. package/docs/INSTALLATION.md +7 -1
  19. package/docs/KNOWN-ISSUES.md +1 -1
  20. package/docs/PROOF-PASSPORT.md +21 -0
  21. package/docs/WORKSPACES.md +27 -0
  22. package/docs/cost-controls.md +74 -12
  23. package/docs/dev/architecture-reference.md +2 -2
  24. package/docs/environment-variables.md +7 -0
  25. package/docs/merge-queue.md +44 -3
  26. package/docs/sentry-intake.md +1 -1
  27. package/docs/slack.md +1 -1
  28. package/docs/v10/GUIDE.md +22 -4
  29. package/loki-ts/dist/loki.js +1502 -1456
  30. package/mcp/__init__.py +1 -1
  31. package/memory/knowledge_graph.py +442 -56
  32. package/package.json +2 -2
  33. package/packages/control-plane/dist/server.js +2182 -1808
  34. package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
  35. package/schemas/loki-yaml.schema.json +1 -0
  36. package/web-app/dist/assets/{AdminPage-DU19V0Z8.js → AdminPage-uxbsIyND.js} +1 -1
  37. package/web-app/dist/assets/{Avatar-8tZOYCzr.js → Avatar-5GhUf4pP.js} +1 -1
  38. package/web-app/dist/assets/{Badge-WsBr_wmd.js → Badge-93rdLyl8.js} +1 -1
  39. package/web-app/dist/assets/{Button-dbsRjgbK.js → Button-C94f2_e6.js} +1 -1
  40. package/web-app/dist/assets/{CockpitPage-BdmKE1BD.js → CockpitPage-C8cDx1mp.js} +1 -1
  41. package/web-app/dist/assets/{ComparePage-B0baIWVY.js → ComparePage-BU7VVcVf.js} +1 -1
  42. package/web-app/dist/assets/{ErrorBoundary-KGmneVKo.js → ErrorBoundary-Bn81ZPkG.js} +1 -1
  43. package/web-app/dist/assets/{EvidenceReceiptPanel-DzbcVDL0.js → EvidenceReceiptPanel-CVqzhxXI.js} +1 -1
  44. package/web-app/dist/assets/{GitHubIssuesPanel-BO6bSgxF.js → GitHubIssuesPanel-i1Yjc0mH.js} +1 -1
  45. package/web-app/dist/assets/{GitHubPRsPanel-BYiyPZye.js → GitHubPRsPanel-eoXw0hiw.js} +1 -1
  46. package/web-app/dist/assets/{HomePage-DDVp5P4l.js → HomePage-Cv_BXFF9.js} +2 -2
  47. package/web-app/dist/assets/{LoginPage-CSMf6PA9.js → LoginPage-CWYdAQSQ.js} +1 -1
  48. package/web-app/dist/assets/{MagicPage-CHxAoj4t.js → MagicPage-5r2ci7MS.js} +1 -1
  49. package/web-app/dist/assets/{MetricsPage-BgSvqiiI.js → MetricsPage-C37jE1uR.js} +1 -1
  50. package/web-app/dist/assets/{NotFoundPage-DMC9Etyr.js → NotFoundPage-CuBrziZd.js} +1 -1
  51. package/web-app/dist/assets/{ProjectPage-BYpWEnfs.js → ProjectPage-59GLzJlR.js} +3 -3
  52. package/web-app/dist/assets/{ProjectsPage-DksZuUPc.js → ProjectsPage-DgDtFRNm.js} +1 -1
  53. package/web-app/dist/assets/{SettingsPage-BRghQ4OZ.js → SettingsPage-DjMSy-kZ.js} +1 -1
  54. package/web-app/dist/assets/{ShowcasePage-AX3TFZdD.js → ShowcasePage-DESkYmrG.js} +1 -1
  55. package/web-app/dist/assets/{SystemSettingsPage-DVg5oEf3.js → SystemSettingsPage-d5jFreyF.js} +1 -1
  56. package/web-app/dist/assets/{TeamsPage-CNKKrZdS.js → TeamsPage-BEVAkxj3.js} +1 -1
  57. package/web-app/dist/assets/{TemplatesPage-DReGZVmw.js → TemplatesPage-DeK9wx22.js} +1 -1
  58. package/web-app/dist/assets/{TerminalOutput-6g4gZVbt.js → TerminalOutput-2vwck437.js} +1 -1
  59. package/web-app/dist/assets/{activity-kKs9rNVl.js → activity-XgL1xWCM.js} +1 -1
  60. package/web-app/dist/assets/{bell-D2f6TSrY.js → bell-EXDsjrzh.js} +1 -1
  61. package/web-app/dist/assets/{bot-CTiUgNPd.js → bot-B9I58zKH.js} +1 -1
  62. package/web-app/dist/assets/{check-b7nwLuDG.js → check-DgLUYM2x.js} +1 -1
  63. package/web-app/dist/assets/{chevron-left-z43N9oQm.js → chevron-left-BhHLpt7K.js} +1 -1
  64. package/web-app/dist/assets/{circle-alert-CFXh7q4_.js → circle-alert-rgioMmC9.js} +1 -1
  65. package/web-app/dist/assets/{clock-C5XlY5gQ.js → clock-BO__fq1n.js} +1 -1
  66. package/web-app/dist/assets/{cloud-DiRynGU2.js → cloud-iLJ3RtJs.js} +1 -1
  67. package/web-app/dist/assets/{code-xml-D0-7LFV7.js → code-xml-BFYU6YlU.js} +1 -1
  68. package/web-app/dist/assets/{database-BL7633cr.js → database-CXDLeUh_.js} +1 -1
  69. package/web-app/dist/assets/{dollar-sign-DYcrnk0S.js → dollar-sign-C9Jbgo-O.js} +1 -1
  70. package/web-app/dist/assets/{file-code-corner-CUf4Ao24.js → file-code-corner-CNPi6dkK.js} +1 -1
  71. package/web-app/dist/assets/{file-plus-DiPFi2oQ.js → file-plus-DAc0Sd1J.js} +1 -1
  72. package/web-app/dist/assets/{globe-2vq90XD6.js → globe-DN0njo3S.js} +1 -1
  73. package/web-app/dist/assets/{hammer-CPLZcW5e.js → hammer-CnpZO-SH.js} +1 -1
  74. package/web-app/dist/assets/{index-8TqwI_NF.js → index-BHwx0KjF.js} +3 -3
  75. package/web-app/dist/assets/{layers-DkjwiuGI.js → layers-_kP2XVgZ.js} +1 -1
  76. package/web-app/dist/assets/{loader-circle-B6seK0mv.js → loader-circle-CcupfVc5.js} +1 -1
  77. package/web-app/dist/assets/{lock-BokWI7Bq.js → lock-D3I89F_A.js} +1 -1
  78. package/web-app/dist/assets/{package-Bwj3Yy9P.js → package-BMz7_qR6.js} +1 -1
  79. package/web-app/dist/assets/{plus-DDtdbe28.js → plus-3rKyjjc7.js} +1 -1
  80. package/web-app/dist/assets/{refresh-cw-MCFH-IYf.js → refresh-cw-DGI9P9uf.js} +1 -1
  81. package/web-app/dist/assets/{rotate-ccw-DpGrHe4v.js → rotate-ccw-CEIZqe5z.js} +1 -1
  82. package/web-app/dist/assets/{scroll-text-AlMeqOJW.js → scroll-text-CBQ5ISFk.js} +1 -1
  83. package/web-app/dist/assets/{server-DlQAxyoR.js → server-DOr9ns7A.js} +1 -1
  84. package/web-app/dist/assets/{shield-alert-QMLIe6EO.js → shield-alert-pcQbbyMb.js} +1 -1
  85. package/web-app/dist/assets/{thumbs-up-CH-iPy2L.js → thumbs-up-Q7fwPWa9.js} +1 -1
  86. package/web-app/dist/assets/{trash-2-CC40OFfT.js → trash-2-D81689SS.js} +1 -1
  87. package/web-app/dist/assets/{trending-down-BkpkaJJX.js → trending-down-8b0O5hb9.js} +1 -1
  88. package/web-app/dist/assets/{trending-up-M8yk4S0A.js → trending-up-CD0tv_w5.js} +1 -1
  89. package/web-app/dist/assets/{upload-EHu6t2yj.js → upload-CphI5Pe_.js} +1 -1
  90. package/web-app/dist/assets/{usePolling-BavtK4H1.js → usePolling-yhGOhxnY.js} +1 -1
  91. package/web-app/dist/assets/{user-DP41l_sQ.js → user-CXM61B6o.js} +1 -1
  92. package/web-app/dist/index.html +1 -1
@@ -49,6 +49,7 @@ import os
49
49
  import queue
50
50
  import re
51
51
  import secrets
52
+ import shutil
52
53
  import signal
53
54
  import socket
54
55
  import socketserver
@@ -343,7 +344,6 @@ def load_config():
343
344
  "port": 7373,
344
345
  "secret": "",
345
346
  "dry_run": False,
346
- "enabled_events": ["issues", "pull_request", "workflow_run"],
347
347
  "workers": DEFAULT_WORKERS,
348
348
  "queue_size": DEFAULT_QUEUE_SIZE,
349
349
  }
@@ -414,6 +414,132 @@ def log_event(event_type, action, payload_summary, status):
414
414
  f.write(line)
415
415
 
416
416
 
417
+ DURATIONS_KEPT = 50
418
+
419
+
420
+ class TriggerHealth:
421
+ """Per-source run counters and auto-pause, persisted in .loki/triggers/health.json.
422
+
423
+ Every mutation re-reads the file under one lock and replaces it atomically, so
424
+ a `loki trigger resume` run between two events is honored. For `start` runs an
425
+ exit code 0 means VERIFIED or ALREADY_SATISFIED.
426
+ """
427
+
428
+ def __init__(self):
429
+ self._lock = threading.Lock()
430
+
431
+ @staticmethod
432
+ def pause_after():
433
+ raw = os.environ.get("LOKI_TRIGGER_PAUSE_AFTER", "3")
434
+ try:
435
+ n = int(raw)
436
+ except ValueError:
437
+ return 3
438
+ return n if n >= 0 else 3
439
+
440
+ def _path(self):
441
+ return get_loki_dir() / "health.json"
442
+
443
+ def _load(self):
444
+ try:
445
+ data = json.loads(self._path().read_text(encoding="utf-8"))
446
+ except (OSError, ValueError):
447
+ return {}
448
+ return data if isinstance(data, dict) else {}
449
+
450
+ def _save(self, data):
451
+ path = self._path()
452
+ tmp = path.with_name("health.json.%d.tmp" % os.getpid())
453
+ tmp.write_text(json.dumps(data, indent=1), encoding="utf-8")
454
+ os.replace(tmp, path)
455
+
456
+ @staticmethod
457
+ def _entry(data, source):
458
+ e = data.get(source)
459
+ if not isinstance(e, dict):
460
+ e = {}
461
+ e.setdefault("runs", 0)
462
+ e.setdefault("passed", 0)
463
+ e.setdefault("consecutive_failures", 0)
464
+ e.setdefault("paused", False)
465
+ e.setdefault("paused_at", "")
466
+ if not isinstance(e.get("durations_s"), list):
467
+ e["durations_s"] = []
468
+ data[source] = e
469
+ return e
470
+
471
+ def record(self, source, returncode, elapsed_s):
472
+ if returncode in (130, 143):
473
+ return
474
+ newly_paused = 0
475
+ with self._lock:
476
+ data = self._load()
477
+ e = self._entry(data, source)
478
+ e["runs"] += 1
479
+ if returncode == 0:
480
+ e["passed"] += 1
481
+ e["consecutive_failures"] = 0
482
+ e["durations_s"] = (e["durations_s"] + [round(elapsed_s, 1)])[-DURATIONS_KEPT:]
483
+ else:
484
+ e["consecutive_failures"] += 1
485
+ limit = self.pause_after()
486
+ if (limit > 0 and e["consecutive_failures"] >= limit
487
+ and source not in AUTOPAUSE_EXEMPT and not e["paused"]):
488
+ e["paused"] = True
489
+ e["paused_at"] = _utc_now()
490
+ newly_paused = e["consecutive_failures"]
491
+ try:
492
+ self._save(data)
493
+ except OSError as err:
494
+ logging.error("trigger health write failed: %s", err)
495
+ if newly_paused:
496
+ log_event(source, "", "%d consecutive failed runs" % newly_paused, "auto-paused")
497
+ send_notification(
498
+ "Trigger source %s paused after %d consecutive failed runs; resume with: loki trigger resume %s"
499
+ % (source, newly_paused, source))
500
+
501
+ def is_paused(self, source):
502
+ with self._lock:
503
+ e = self._load().get(source)
504
+ return bool(isinstance(e, dict) and e.get("paused"))
505
+
506
+ def set_paused(self, source, flag):
507
+ with self._lock:
508
+ data = self._load()
509
+ e = self._entry(data, source)
510
+ e["paused"] = bool(flag)
511
+ e["paused_at"] = _utc_now() if flag else ""
512
+ if not flag:
513
+ e["consecutive_failures"] = 0
514
+ self._save(data)
515
+
516
+ def snapshot(self):
517
+ with self._lock:
518
+ return self._load()
519
+
520
+ def paused_sources(self):
521
+ return sorted(k for k, v in self.snapshot().items() if isinstance(v, dict) and v.get("paused"))
522
+
523
+
524
+ HEALTH = TriggerHealth()
525
+
526
+
527
+ def health_lines():
528
+ """One status line per source that has any record (the lane metric)."""
529
+ snap = HEALTH.snapshot()
530
+ lines = []
531
+ for src in ALL_SOURCES:
532
+ e = snap.get(src)
533
+ if not isinstance(e, dict) or not (e.get("runs") or e.get("paused")):
534
+ continue
535
+ d = sorted(x for x in e.get("durations_s", []) if isinstance(x, (int, float)))
536
+ median = ("%ds" % round(d[len(d) // 2] if len(d) % 2 else (d[len(d) // 2 - 1] + d[len(d) // 2]) / 2)) if d else "n/a"
537
+ state = "PAUSED since %s" % e.get("paused_at") if e.get("paused") else "active"
538
+ lines.append("%s: runs %d, passed %d (exit 0 = VERIFIED for start runs), consecutive failures %d, median %s trigger-to-exit, %s"
539
+ % (src, e.get("runs", 0), e.get("passed", 0), e.get("consecutive_failures", 0), median, state))
540
+ return lines
541
+
542
+
417
543
  def validate_signature(secret, body, signature_header):
418
544
  """Validate GitHub HMAC-SHA256 webhook signature (constant-time).
419
545
 
@@ -471,7 +597,7 @@ def _remove_quietly(*paths):
471
597
  pass
472
598
 
473
599
 
474
- def _reap_child(proc):
600
+ def _reap_child(proc, on_exit=None):
475
601
  """Wait on a child process so it is always reaped (no zombie).
476
602
 
477
603
  Used when the dispatch outlives our synchronous wait window: instead of
@@ -485,7 +611,12 @@ def _reap_child(proc):
485
611
  try:
486
612
  proc.wait()
487
613
  except Exception:
488
- pass
614
+ return
615
+ if on_exit is not None:
616
+ try:
617
+ on_exit(proc.returncode)
618
+ except Exception as e:
619
+ logging.error("on_exit callback failed: %s", e)
489
620
 
490
621
 
491
622
  # Outcome of a dispatch, as distinct from "did it launch".
@@ -527,13 +658,24 @@ def _scoped_child_env(extra_env=None):
527
658
  return env
528
659
 
529
660
 
530
- def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
661
+ # Commands whose exit code feeds the per-source trigger health counters.
662
+ RUN_COMMANDS = ("start", "verify-pr")
663
+
664
+ # Set by dispatch_event on the worker thread: the trigger source of the event
665
+ # being handled, so run_loki_outcome can attribute the run's exit code to it.
666
+ _DISPATCH = threading.local()
667
+
668
+
669
+ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None, cwd=None, on_exit=None):
531
670
  """Run a loki command and report WHICH of the three outcomes occurred.
532
671
 
533
672
  Returns one of OUTCOME_PASSED / OUTCOME_FAILED / OUTCOME_UNKNOWN. This is
534
673
  the honest version of run_loki_command, which answers only "did it launch".
535
674
  When stdout_sink is a list, the decoded stdout is appended to it on completion.
536
675
  extra_env adds child env vars (e.g. LOKI_STEER_SOURCE) on top of the scoped env.
676
+ on_exit(returncode) is called once the child ends, also from the reaper thread
677
+ for a detached run; it gets None when the launch itself failed. A start or
678
+ verify-pr run launched for a trigger source is also recorded in HEALTH.
537
679
  """
538
680
  cmd = ["loki"] + args
539
681
  if dry_run:
@@ -543,6 +685,15 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
543
685
  # Child output goes to per-run files, never a pipe: past the wait window
544
686
  # nothing reads a pipe, so a child writing more than the ~64KB pipe buffer
545
687
  # would block forever. The files double as the run log for a detached child.
688
+ source = getattr(_DISPATCH, "source", None)
689
+ if source and args and args[0] in RUN_COMMANDS:
690
+ t0 = time.monotonic()
691
+ user_cb = on_exit
692
+
693
+ def on_exit(rc, _src=source, _t0=t0, _cb=user_cb):
694
+ HEALTH.record(_src, rc, time.monotonic() - _t0)
695
+ if _cb is not None:
696
+ _cb(rc)
546
697
  out_path, err_path = _dispatch_log_paths()
547
698
  try:
548
699
  with open(out_path, "wb") as out_f, open(err_path, "wb") as err_f:
@@ -551,10 +702,13 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
551
702
  stdout=out_f,
552
703
  stderr=err_f,
553
704
  env=_scoped_child_env(extra_env),
705
+ cwd=cwd,
554
706
  )
555
707
  except (FileNotFoundError, OSError) as e:
556
708
  logging.error("Failed to launch %s: %s", " ".join(cmd), e)
557
709
  _remove_quietly(out_path, err_path)
710
+ if on_exit is not None:
711
+ on_exit(None)
558
712
  return OUTCOME_FAILED
559
713
 
560
714
  try:
@@ -571,7 +725,7 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
571
725
  )
572
726
  threading.Thread(
573
727
  target=_reap_child,
574
- args=(proc,),
728
+ args=(proc, on_exit),
575
729
  name="loki-trigger-reaper-%d" % proc.pid,
576
730
  daemon=True,
577
731
  ).start()
@@ -582,6 +736,11 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
582
736
  _remove_quietly(out_path, err_path)
583
737
  if stdout_sink is not None:
584
738
  stdout_sink.append(stdout.decode("utf-8", errors="replace"))
739
+ if on_exit is not None:
740
+ try:
741
+ on_exit(proc.returncode)
742
+ except Exception as e:
743
+ logging.error("on_exit callback failed: %s", e)
585
744
 
586
745
  if proc.returncode == 0:
587
746
  logging.info("Dispatch pid=%d completed (exit 0)", proc.pid)
@@ -599,7 +758,7 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
599
758
  return OUTCOME_FAILED
600
759
 
601
760
 
602
- def run_loki_command(args, dry_run=False, stdout_sink=None):
761
+ def run_loki_command(args, dry_run=False, stdout_sink=None, cwd=None, on_exit=None):
603
762
  """Run a loki command synchronously and reap it; or print it if dry_run.
604
763
 
605
764
  Returns True if the command was launched and exited 0 (or backgrounded
@@ -615,7 +774,7 @@ def run_loki_command(args, dry_run=False, stdout_sink=None):
615
774
  passed wants run_loki_outcome instead. Behaviour is unchanged: a detached
616
775
  dispatch (UNKNOWN) still reads as True here, exactly as before.
617
776
  """
618
- return run_loki_outcome(args, dry_run=dry_run, stdout_sink=stdout_sink) in (
777
+ return run_loki_outcome(args, dry_run=dry_run, stdout_sink=stdout_sink, cwd=cwd, on_exit=on_exit) in (
619
778
  OUTCOME_PASSED, OUTCOME_UNKNOWN,
620
779
  )
621
780
 
@@ -874,8 +1033,33 @@ def handle_pull_request_event(payload, dry_run=False):
874
1033
  return summary, status
875
1034
 
876
1035
 
1036
+ def build_ci_brief(wf, repo):
1037
+ """One-line, control-char-free brief for a red default-branch CI run, or "" if unusable."""
1038
+ run_id = wf.get("id")
1039
+ if not isinstance(run_id, int) or isinstance(run_id, bool) or run_id <= 0:
1040
+ return ""
1041
+ sha = str(wf.get("head_sha") or "")
1042
+ sha7 = sha[:7] if re.fullmatch(r"[0-9a-fA-F]{7,40}", sha) else "unknown"
1043
+ url = str(wf.get("html_url") or "")
1044
+ where = "run %d" % run_id
1045
+ if url.startswith("https://github.com/"):
1046
+ where += ", %s" % _clip(url, 200)
1047
+ brief = (
1048
+ "CI is red on the default branch of %s: workflow '%s' (%s) failed at commit %s on %s. "
1049
+ "Read the failure with `gh run view %d --repo %s --log-failed`, reproduce it locally, "
1050
+ "fix the root cause with the smallest change, and keep every existing test passing. "
1051
+ "Do not skip, disable or delete tests or CI steps."
1052
+ % (repo, _clip(wf.get("name")), where, sha7, _clip(wf.get("head_branch")), run_id, repo))
1053
+ brief = " ".join(CONTROL_CHARS_RE.sub(" ", brief).split())
1054
+ return brief.encode("utf-8")[:SENTRY_BRIEF_MAX].decode("utf-8", errors="ignore")
1055
+
1056
+
877
1057
  def handle_workflow_run_event(payload, dry_run=False):
878
- """Handle workflow_run event: completed+failure -> loki start with context."""
1058
+ """Handle workflow_run: red default-branch CI -> loki start --brief <CI brief> --pr.
1059
+
1060
+ Only the default branch: a red PR branch is `loki merge watch`'s job, and Loki's
1061
+ own loki/* branches must never retrigger themselves.
1062
+ """
879
1063
  action = payload.get("action", "")
880
1064
  if action != "completed":
881
1065
  return None, "skipped (action=%s)" % action
@@ -883,14 +1067,22 @@ def handle_workflow_run_event(payload, dry_run=False):
883
1067
  conclusion = wf.get("conclusion", "")
884
1068
  if conclusion != "failure":
885
1069
  return None, "skipped (conclusion=%s)" % conclusion
886
- wf_name = wf.get("name", "unknown")
887
1070
  repo = payload.get("repository", {})
888
1071
  repo_full_name = repo.get("full_name", "")
889
- summary = "workflow '%s' failed in %s" % (wf_name, repo_full_name)
890
- # CI-failure context: re-run the current spec in the working directory so
891
- # the agent can repair the failing build. No issue/PR ref to attach here.
892
- args = ["start"]
893
- success = run_loki_command(args, dry_run=dry_run)
1072
+ if not valid_repo_full_name(repo_full_name):
1073
+ return None, "rejected (invalid repository name)"
1074
+ default = repo.get("default_branch", "")
1075
+ if not default:
1076
+ return None, "skipped (no default_branch in payload)"
1077
+ branch = str(wf.get("head_branch") or "")
1078
+ if branch != default or branch.startswith("loki/"):
1079
+ return None, ("skipped (failure on branch %s, not the default branch; "
1080
+ "use loki merge watch for PR branches)" % _clip(branch))
1081
+ brief = build_ci_brief(wf, repo_full_name)
1082
+ if not brief or not valid_spec(brief):
1083
+ return None, "rejected (invalid brief)"
1084
+ summary = "workflow '%s' failed in %s" % (_clip(wf.get("name", "unknown")), repo_full_name)
1085
+ success = run_loki_command(brief_start_args(brief), dry_run=dry_run)
894
1086
  status = "fired" if success else "error"
895
1087
  if success:
896
1088
  send_notification("Trigger fired: CI failure - %s" % summary)
@@ -1398,17 +1590,126 @@ def handle_sentry_event(payload, dry_run=False):
1398
1590
  return summary, status
1399
1591
 
1400
1592
 
1593
+ REVIEW_REF_RE = re.compile(r"^[A-Za-z0-9._][A-Za-z0-9._/-]{0,199}$")
1594
+ FULL_SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$")
1595
+ REVIEW_CLONE_TIMEOUT = 120
1596
+
1597
+
1598
+ def _review_root():
1599
+ return (get_loki_dir() / "review").resolve()
1600
+
1601
+
1602
+ def _remove_review_clone(path):
1603
+ """Delete a review clone, only when it sits under the review root."""
1604
+ try:
1605
+ real = Path(path).resolve()
1606
+ if real.parent == _review_root():
1607
+ shutil.rmtree(real, ignore_errors=True)
1608
+ except OSError as e:
1609
+ logging.error("review clone cleanup failed: %s", e)
1610
+
1611
+
1612
+ def handle_pull_request_review_event(payload, dry_run=False):
1613
+ """A trusted 'changes requested' review runs `loki verify-pr respond` on a fresh clone of the PR branch."""
1614
+ action = payload.get("action", "")
1615
+ if action != "submitted":
1616
+ return None, "skipped (action=%s)" % action
1617
+ review = payload.get("review") or {}
1618
+ state = str(review.get("state", "")).lower()
1619
+ if state != "changes_requested":
1620
+ return None, "skipped (review state=%s)" % _clip(state, 30)
1621
+ if review.get("author_association") not in TRUSTED_ASSOCIATIONS:
1622
+ return None, "skipped (review author not authorized)"
1623
+ user = review.get("user") or {}
1624
+ if user.get("type") == "Bot" or str(user.get("login", "")).endswith("[bot]"):
1625
+ return None, "skipped (bot review)"
1626
+ if LOKI_COMMENT_MARKER in str(review.get("body") or ""):
1627
+ return None, "skipped (loki-authored review)"
1628
+ pr = payload.get("pull_request") or {}
1629
+ repo_full_name = (payload.get("repository") or {}).get("full_name", "")
1630
+ if not valid_repo_full_name(repo_full_name):
1631
+ return None, "rejected (invalid repository full_name)"
1632
+ number = pr.get("number")
1633
+ if not valid_issue_number(number):
1634
+ return None, "rejected (invalid pull request number)"
1635
+ if pr.get("state") != "open":
1636
+ return None, "skipped (pull request not open)"
1637
+ head = pr.get("head") or {}
1638
+ if (head.get("repo") or {}).get("full_name") != repo_full_name:
1639
+ return None, "skipped (pull request head is a fork)"
1640
+ ref = str(head.get("ref") or "")
1641
+ if not REVIEW_REF_RE.match(ref) or ".." in ref:
1642
+ return None, "rejected (invalid head ref)"
1643
+ sha = str(head.get("sha") or "")
1644
+ if not FULL_SHA_RE.match(sha):
1645
+ return None, "rejected (invalid head sha)"
1646
+ summary = "review changes requested on %s#%s" % (repo_full_name, number)
1647
+ if dry_run:
1648
+ run_loki_command(["verify-pr", "respond", "%s#%s" % (repo_full_name, number)], dry_run=True)
1649
+ return summary, "fired"
1650
+
1651
+ env = _scoped_child_env()
1652
+ try:
1653
+ origin = subprocess.run(["git", "remote", "get-url", "origin"], capture_output=True, text=True,
1654
+ timeout=30, env=env).stdout.strip()
1655
+ except (OSError, subprocess.SubprocessError):
1656
+ origin = ""
1657
+ if not origin:
1658
+ return summary, "error"
1659
+ root = _review_root()
1660
+ root.mkdir(parents=True, exist_ok=True)
1661
+ clone = root / ("pr-%s-%s" % (number, secrets.token_hex(4)))
1662
+ try:
1663
+ subprocess.run(["git", "clone", "--quiet", "--depth", "50", "--single-branch", "--branch", ref,
1664
+ "--", origin, str(clone)], capture_output=True, timeout=REVIEW_CLONE_TIMEOUT,
1665
+ env=env, check=True)
1666
+ got = subprocess.run(["git", "-C", str(clone), "rev-parse", "HEAD"], capture_output=True,
1667
+ text=True, timeout=30, env=env).stdout.strip()
1668
+ except (OSError, subprocess.SubprocessError) as e:
1669
+ logging.error("review clone failed: %s", e)
1670
+ _remove_review_clone(clone)
1671
+ HEALTH.record("review", None, 0.0)
1672
+ return summary, "error"
1673
+ if got.lower() != sha.lower():
1674
+ _remove_review_clone(clone)
1675
+ return None, "skipped (PR head moved)"
1676
+ success = run_loki_command(["verify-pr", "respond", "%s#%s" % (repo_full_name, number)],
1677
+ cwd=str(clone), on_exit=lambda rc: _remove_review_clone(clone))
1678
+ if success:
1679
+ send_notification("Trigger fired: %s" % summary)
1680
+ else:
1681
+ _remove_review_clone(clone)
1682
+ return summary, "fired" if success else "error"
1683
+
1684
+
1401
1685
  # Map event name -> handler. Keeps do_POST routing declarative. JOB_EVENT rides
1402
1686
  # the same table (and therefore the same bounded queue and worker pool) but is
1403
1687
  # explicitly refused on the /webhook path.
1404
- EVENT_HANDLERS = {
1405
- "issues": handle_issues_event,
1406
- "issue_comment": handle_issue_comment_event,
1407
- "pull_request": handle_pull_request_event,
1408
- "workflow_run": handle_workflow_run_event,
1409
- JOB_EVENT: handle_job_event,
1410
- SENTRY_EVENT: handle_sentry_event,
1688
+ # event -> (source name, handler). A None source is steer-only: never counted or paused.
1689
+ TRIGGERS = {
1690
+ "issues": ("issues", handle_issues_event),
1691
+ "issue_comment": (None, handle_issue_comment_event),
1692
+ "pull_request": ("pull_request", handle_pull_request_event),
1693
+ "workflow_run": ("ci", handle_workflow_run_event),
1694
+ "pull_request_review": ("review", handle_pull_request_review_event),
1695
+ JOB_EVENT: ("jobs", handle_job_event),
1696
+ SENTRY_EVENT: ("sentry", handle_sentry_event),
1411
1697
  }
1698
+ EVENT_HANDLERS = {k: v[1] for k, v in TRIGGERS.items()}
1699
+
1700
+ # Sources that share the job queue; _enqueue_job stamps the job with its source.
1701
+ JOB_SOURCES = ("jobs", "slack", "linear", "jira", "teams")
1702
+ ALL_SOURCES = tuple(sorted({v[0] for v in TRIGGERS.values() if v[0]} | set(JOB_SOURCES)))
1703
+ # A direct /jobs caller already gets a per-job failed status and decides for itself.
1704
+ AUTOPAUSE_EXEMPT = ("jobs",)
1705
+
1706
+
1707
+ def trigger_source(event_type, payload):
1708
+ if event_type == JOB_EVENT:
1709
+ src = payload.get("source") if isinstance(payload, dict) else None
1710
+ return src if src in JOB_SOURCES else "jobs"
1711
+ entry = TRIGGERS.get(event_type)
1712
+ return entry[0] if entry else None
1412
1713
 
1413
1714
 
1414
1715
  def dispatch_event(event_type, payload, dry_run=False):
@@ -1423,11 +1724,20 @@ def dispatch_event(event_type, payload, dry_run=False):
1423
1724
  status = "unsupported event: %s" % event_type
1424
1725
  log_event(event_type, action, "", status)
1425
1726
  return None, status
1727
+ source = trigger_source(event_type, payload)
1728
+ if source and HEALTH.is_paused(source):
1729
+ status = "paused (source %s; loki trigger resume %s)" % (source, source)
1730
+ log_event(event_type, action, "", status)
1731
+ # A queued job needs a terminal status so its submitter does not poll forever.
1732
+ return (status, JOB_STATUS_FAILED) if event_type == JOB_EVENT else (None, "paused")
1733
+ _DISPATCH.source = source
1426
1734
  try:
1427
1735
  summary, status = handler(payload, dry_run=dry_run)
1428
1736
  except Exception as e: # defensive: never let a worker die on bad input
1429
1737
  logging.exception("Handler for %s raised: %s", event_type, e)
1430
1738
  summary, status = None, "error"
1739
+ finally:
1740
+ _DISPATCH.source = None
1431
1741
  log_event(event_type, action, summary or "", status)
1432
1742
  return summary, status
1433
1743
 
@@ -1521,9 +1831,24 @@ class Dispatcher:
1521
1831
  """True if the same repo#N + action + head sha was accepted and its run is live.
1522
1832
 
1523
1833
  Catches a redelivery under a new delivery id or after a restart. Once the
1524
- run is no longer live the same key may dispatch again. Only issues and
1525
- pull_request events carry an intake key.
1834
+ run is no longer live the same key may dispatch again. A red workflow_run
1835
+ has no issue ref: one repair run per repo and head sha, across workflows
1836
+ and redeliveries, so its key stays remembered.
1526
1837
  """
1838
+ if event_type == "workflow_run":
1839
+ repo = (payload.get("repository") or {}).get("full_name", "")
1840
+ wf = payload.get("workflow_run") or {}
1841
+ sha = wf.get("head_sha", "")
1842
+ # Only a red completed run takes the key: a green run of the same sha must not mask it.
1843
+ if (payload.get("action") != "completed" or wf.get("conclusion") != "failure"
1844
+ or not repo or not sha):
1845
+ return False
1846
+ key = "intake:workflow_run:%s@%s" % (repo, sha)
1847
+ with self._dedup_lock:
1848
+ if key in self._seen_deliveries:
1849
+ return True
1850
+ self._remember(key)
1851
+ return False
1527
1852
  if event_type not in ("issues", "pull_request"):
1528
1853
  return False
1529
1854
  number = (payload.get("issue") or payload.get("pull_request") or {}).get("number")
@@ -1816,7 +2141,8 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
1816
2141
  "status": "running",
1817
2142
  "dry_run": self.dry_run,
1818
2143
  "port": config.get("port", 7373),
1819
- "enabled_events": config.get("enabled_events", []),
2144
+ "triggers": HEALTH.snapshot(),
2145
+ "paused": HEALTH.paused_sources(),
1820
2146
  "secret_configured": bool(self.secret),
1821
2147
  "api_token_configured": bool(self.api_token),
1822
2148
  })
@@ -2097,14 +2423,14 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
2097
2423
  return
2098
2424
  self._send_json(202, {"id": job_id, "status": "queued"})
2099
2425
 
2100
- def _enqueue_job(self, spec, reply_target=None, pr=False, unit_job=None):
2426
+ def _enqueue_job(self, spec, reply_target=None, pr=False, unit_job=None, source="jobs"):
2101
2427
  """Record and enqueue one validated job; shared by /jobs, /slack, /linear.
2102
2428
 
2103
2429
  Returns the job id, or None after sending 429 when the queue is full.
2104
2430
  reply_target must come from an already-verified request.
2105
2431
  """
2106
2432
  job_id = secrets.token_urlsafe(12)
2107
- job = {"spec": spec, "job_id": job_id}
2433
+ job = {"spec": spec, "job_id": job_id, "source": source}
2108
2434
  if pr:
2109
2435
  job["pr"] = True
2110
2436
  if unit_job is not None:
@@ -2183,7 +2509,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
2183
2509
  target = {"kind": "slack"}
2184
2510
  if response_url.startswith(SLACK_REPLY_PREFIX):
2185
2511
  target["response_url"] = response_url
2186
- job_id = self._enqueue_job(text.strip(), target)
2512
+ job_id = self._enqueue_job(text.strip(), target, source="slack")
2187
2513
  if job_id is None:
2188
2514
  return
2189
2515
  self._send_json(200, {"response_type": "ephemeral",
@@ -2240,7 +2566,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
2240
2566
  issue_id = data.get("id")
2241
2567
  target = {"kind": "linear",
2242
2568
  "issue_id": issue_id if isinstance(issue_id, str) else ""}
2243
- job_id = self._enqueue_job(spec, target)
2569
+ job_id = self._enqueue_job(spec, target, source="linear")
2244
2570
  if job_id is None:
2245
2571
  return
2246
2572
  self._send_json(202, {"id": job_id, "status": "queued"})
@@ -2298,7 +2624,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
2298
2624
  if dup_id or dup_body:
2299
2625
  self._send_json(200, {"status": "duplicate"})
2300
2626
  return
2301
- job_id = self._enqueue_job(brief, {"kind": "jira", "key": key}, pr=True)
2627
+ job_id = self._enqueue_job(brief, {"kind": "jira", "key": key}, pr=True, source="jira")
2302
2628
  if job_id is None:
2303
2629
  return
2304
2630
  self._send_json(202, {"id": job_id, "status": "queued"})
@@ -2339,7 +2665,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
2339
2665
  if self.dispatcher.seen_delivery(teams_dedup_key(activity)):
2340
2666
  self._send_json(200, {"type": "message", "text": "Duplicate, already queued."})
2341
2667
  return
2342
- job_id = self._enqueue_job(brief, pr=True)
2668
+ job_id = self._enqueue_job(brief, pr=True, source="teams")
2343
2669
  if job_id is None:
2344
2670
  return
2345
2671
  self._send_json(200, {"type": "message", "text": "Queued loki job %s" % job_id})
@@ -2544,8 +2870,23 @@ def main():
2544
2870
  parser.add_argument("--dry-run", action="store_true", help="Preview triggers without running loki")
2545
2871
  parser.add_argument("--workers", type=int, default=None, help="Worker threads draining the dispatch queue")
2546
2872
  parser.add_argument("--queue-size", type=int, default=None, help="Max in-flight queued dispatches")
2873
+ ctl = parser.add_mutually_exclusive_group()
2874
+ ctl.add_argument("--health", action="store_true", help="Print per-source run counts and pause state, then exit")
2875
+ ctl.add_argument("--resume", metavar="SOURCE", help="Resume a paused trigger source, then exit")
2547
2876
  args = parser.parse_args()
2548
2877
 
2878
+ if args.health:
2879
+ lines = health_lines()
2880
+ print("\n".join(lines) if lines else "No triggered runs recorded yet.")
2881
+ return 0
2882
+ if args.resume is not None:
2883
+ if args.resume not in ALL_SOURCES:
2884
+ print("unknown trigger source: %s (known: %s)" % (args.resume, ", ".join(ALL_SOURCES)))
2885
+ return 2
2886
+ HEALTH.set_paused(args.resume, False)
2887
+ print("%s: resumed" % args.resume)
2888
+ return 0
2889
+
2549
2890
  logging.basicConfig(
2550
2891
  level=logging.INFO,
2551
2892
  format="%(asctime)s [%(levelname)s] %(message)s",
@@ -2712,7 +3053,7 @@ if __name__ == "__main__":
2712
3053
  if hasattr(signal, "SIGPIPE"):
2713
3054
  signal.signal(signal.SIGPIPE, signal.SIG_DFL)
2714
3055
 
2715
- main()
3056
+ rc = main()
2716
3057
 
2717
3058
  # ponytail: flush stdout/stderr inside try/except BrokenPipeError before
2718
3059
  # exit, then call os._exit() to avoid daemon thread cleanup during
@@ -2725,4 +3066,4 @@ if __name__ == "__main__":
2725
3066
  sys.stderr.flush()
2726
3067
  except BrokenPipeError:
2727
3068
  pass
2728
- os._exit(0)
3069
+ os._exit(rc if isinstance(rc, int) else 0)