loki-mode 11.128.2 → 11.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/SKILL.md +3 -3
- package/VERSION +1 -1
- package/autonomy/lib/help-core.sh +30 -42
- package/autonomy/lib/workspace.py +292 -19
- package/autonomy/lib/workspace_comment.py +40 -11
- package/autonomy/loki +129 -159
- package/autonomy/route.sh +3 -2
- package/autonomy/trigger-server.py +373 -32
- package/bin/loki +139 -18
- package/completions/_loki +11 -142
- package/completions/loki.bash +4 -2
- package/dashboard/__init__.py +1 -1
- package/docs/CLI-CORE-VERBS.md +61 -37
- package/docs/CLI-INVENTORY.md +152 -153
- package/docs/CLI-REFERENCE.md +20 -166
- package/docs/DEPLOYMENT.md +38 -3
- package/docs/INSTALLATION.md +7 -1
- package/docs/KNOWN-ISSUES.md +1 -1
- package/docs/PROOF-PASSPORT.md +21 -0
- package/docs/TESTING.md +1 -1
- package/docs/WORKSPACES.md +27 -0
- package/docs/cost-controls.md +74 -12
- package/docs/dev/architecture-reference.md +2 -2
- package/docs/environment-variables.md +7 -0
- package/docs/exit-codes.md +7 -0
- package/docs/merge-queue.md +44 -3
- package/docs/sentry-intake.md +1 -1
- package/docs/slack.md +1 -1
- package/docs/v10/GUIDE.md +33 -4
- package/loki-ts/dist/loki.js +1695 -1649
- package/mcp/__init__.py +1 -1
- package/memory/knowledge_graph.py +442 -56
- package/package.json +3 -3
- package/packages/control-plane/dist/server.js +2166 -1633
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/schemas/loki-yaml.schema.json +1 -0
- package/web-app/dist/assets/{AdminPage-B-9a_urH.js → AdminPage-CNxvd-uy.js} +1 -1
- package/web-app/dist/assets/{Avatar-DWFQPBje.js → Avatar-DdLoiZKv.js} +1 -1
- package/web-app/dist/assets/{Badge-CuLYjGKc.js → Badge-DrdPaBOH.js} +1 -1
- package/web-app/dist/assets/{Button-Cws5TOeA.js → Button-BQ0UyQAk.js} +1 -1
- package/web-app/dist/assets/{CockpitPage-CCT1U0rq.js → CockpitPage-BLFBjBZW.js} +1 -1
- package/web-app/dist/assets/{ComparePage-ChcsGaeo.js → ComparePage-NHaUjY-d.js} +1 -1
- package/web-app/dist/assets/{ErrorBoundary-C5Mz6Y_d.js → ErrorBoundary-ClaDCylr.js} +1 -1
- package/web-app/dist/assets/{EvidenceReceiptPanel-jXRFyeG-.js → EvidenceReceiptPanel-BBW0IhXb.js} +1 -1
- package/web-app/dist/assets/{GitHubIssuesPanel-C_YjFJSB.js → GitHubIssuesPanel-B_dAtNuN.js} +1 -1
- package/web-app/dist/assets/{GitHubPRsPanel-B3bRY_lb.js → GitHubPRsPanel-Tv3JF9E0.js} +1 -1
- package/web-app/dist/assets/{HomePage-JLwBDYu8.js → HomePage-8fLInImK.js} +2 -2
- package/web-app/dist/assets/{LoginPage-BcDvWJr1.js → LoginPage-BGL-xm0l.js} +1 -1
- package/web-app/dist/assets/{MagicPage-Bj3qzygY.js → MagicPage-D7nHpNuT.js} +1 -1
- package/web-app/dist/assets/{MetricsPage-9k0NVtge.js → MetricsPage-BZh8ZOug.js} +1 -1
- package/web-app/dist/assets/{NotFoundPage-D81Z_SCM.js → NotFoundPage-DJ5YwC3-.js} +1 -1
- package/web-app/dist/assets/{ProjectPage-D9CkJI0B.js → ProjectPage-DYaGNgFt.js} +3 -3
- package/web-app/dist/assets/{ProjectsPage-vxSmYYzh.js → ProjectsPage-9GbGTW3g.js} +1 -1
- package/web-app/dist/assets/{SettingsPage-Cd2AS1Sg.js → SettingsPage-DdRI8LZ5.js} +1 -1
- package/web-app/dist/assets/{ShowcasePage-DOZCRLg5.js → ShowcasePage-ClEeMHAu.js} +1 -1
- package/web-app/dist/assets/{SystemSettingsPage-wl2UO5DF.js → SystemSettingsPage-rqj2byaP.js} +1 -1
- package/web-app/dist/assets/{TeamsPage-CA9FPLPm.js → TeamsPage-OaJ999dm.js} +1 -1
- package/web-app/dist/assets/{TemplatesPage-DLRTBM8i.js → TemplatesPage-CrqXt5pd.js} +1 -1
- package/web-app/dist/assets/{TerminalOutput-RqmPEs4o.js → TerminalOutput-DzOk3P-J.js} +1 -1
- package/web-app/dist/assets/{activity-CoyjHBZ4.js → activity-C3XPvqFU.js} +1 -1
- package/web-app/dist/assets/{bell-Dn2BCsQu.js → bell-BADR28Ic.js} +1 -1
- package/web-app/dist/assets/{bot-CNqPuD8h.js → bot-BOfl_osT.js} +1 -1
- package/web-app/dist/assets/{check-BWAwFpe2.js → check-DyeRhpqt.js} +1 -1
- package/web-app/dist/assets/{chevron-left-BgTjvxLB.js → chevron-left-DytHCita.js} +1 -1
- package/web-app/dist/assets/{circle-alert-CIYxFeDp.js → circle-alert-ej7tR0w9.js} +1 -1
- package/web-app/dist/assets/{clock-Dj6DRvL7.js → clock-CMmgiaJ3.js} +1 -1
- package/web-app/dist/assets/{cloud-DZBvW6-f.js → cloud-BmoOA5GK.js} +1 -1
- package/web-app/dist/assets/{code-xml-B0sAmzAO.js → code-xml-DjcElMOx.js} +1 -1
- package/web-app/dist/assets/{database-BkO44OAR.js → database-BxK55gai.js} +1 -1
- package/web-app/dist/assets/{dollar-sign-ChTZgVxB.js → dollar-sign-MxE9EOHB.js} +1 -1
- package/web-app/dist/assets/{file-code-corner-CQSIiG1T.js → file-code-corner-BYQ975Vf.js} +1 -1
- package/web-app/dist/assets/{file-plus-BAV2KC_M.js → file-plus-C0_fKf6I.js} +1 -1
- package/web-app/dist/assets/{globe-TUNeqVA9.js → globe-C7iJxA9K.js} +1 -1
- package/web-app/dist/assets/{hammer-CMD4wrup.js → hammer-CuVGrR3Z.js} +1 -1
- package/web-app/dist/assets/{index-B5lRV-ZJ.js → index-DvGbCL2t.js} +3 -3
- package/web-app/dist/assets/{layers-BxXkAn6j.js → layers-BWmxXnEV.js} +1 -1
- package/web-app/dist/assets/{loader-circle-CE46Yvcr.js → loader-circle-BrsqCmNy.js} +1 -1
- package/web-app/dist/assets/{lock-3uBPp5eV.js → lock-Dl3UsTMr.js} +1 -1
- package/web-app/dist/assets/{package-DjPZ3DLs.js → package-CCiRYdmb.js} +1 -1
- package/web-app/dist/assets/{plus-D8v6ZCYM.js → plus-d5BugnNw.js} +1 -1
- package/web-app/dist/assets/{refresh-cw-BSPJHBul.js → refresh-cw-DIkE6C0k.js} +1 -1
- package/web-app/dist/assets/{rotate-ccw-D0u6PH4j.js → rotate-ccw-DDhYD9MA.js} +1 -1
- package/web-app/dist/assets/{scroll-text-Dcwu9apa.js → scroll-text-jAoSqO2O.js} +1 -1
- package/web-app/dist/assets/{server-pd5YWyJw.js → server-D16MyCdB.js} +1 -1
- package/web-app/dist/assets/{shield-alert-C3TqM2Xm.js → shield-alert-C8W_Qcb5.js} +1 -1
- package/web-app/dist/assets/{thumbs-up-U1Wkm_Rg.js → thumbs-up-DdTbl2Vo.js} +1 -1
- package/web-app/dist/assets/{trash-2-BspH7NSu.js → trash-2-R-2Gvy6B.js} +1 -1
- package/web-app/dist/assets/{trending-down-DNg_ybr3.js → trending-down-Dk08-nlD.js} +1 -1
- package/web-app/dist/assets/{trending-up-CYzIu-lq.js → trending-up-DX8yONXb.js} +1 -1
- package/web-app/dist/assets/{upload-ChhyDeon.js → upload-6oDG_uiK.js} +1 -1
- package/web-app/dist/assets/{usePolling-ClMveR0q.js → usePolling-Co4817xB.js} +1 -1
- package/web-app/dist/assets/{user-BlFXAv0-.js → user-CjbWWW1W.js} +1 -1
- package/web-app/dist/index.html +1 -1
|
@@ -49,6 +49,7 @@ import os
|
|
|
49
49
|
import queue
|
|
50
50
|
import re
|
|
51
51
|
import secrets
|
|
52
|
+
import shutil
|
|
52
53
|
import signal
|
|
53
54
|
import socket
|
|
54
55
|
import socketserver
|
|
@@ -343,7 +344,6 @@ def load_config():
|
|
|
343
344
|
"port": 7373,
|
|
344
345
|
"secret": "",
|
|
345
346
|
"dry_run": False,
|
|
346
|
-
"enabled_events": ["issues", "pull_request", "workflow_run"],
|
|
347
347
|
"workers": DEFAULT_WORKERS,
|
|
348
348
|
"queue_size": DEFAULT_QUEUE_SIZE,
|
|
349
349
|
}
|
|
@@ -414,6 +414,132 @@ def log_event(event_type, action, payload_summary, status):
|
|
|
414
414
|
f.write(line)
|
|
415
415
|
|
|
416
416
|
|
|
417
|
+
DURATIONS_KEPT = 50
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
class TriggerHealth:
|
|
421
|
+
"""Per-source run counters and auto-pause, persisted in .loki/triggers/health.json.
|
|
422
|
+
|
|
423
|
+
Every mutation re-reads the file under one lock and replaces it atomically, so
|
|
424
|
+
a `loki trigger resume` run between two events is honored. For `start` runs an
|
|
425
|
+
exit code 0 means VERIFIED or ALREADY_SATISFIED.
|
|
426
|
+
"""
|
|
427
|
+
|
|
428
|
+
def __init__(self):
|
|
429
|
+
self._lock = threading.Lock()
|
|
430
|
+
|
|
431
|
+
@staticmethod
|
|
432
|
+
def pause_after():
|
|
433
|
+
raw = os.environ.get("LOKI_TRIGGER_PAUSE_AFTER", "3")
|
|
434
|
+
try:
|
|
435
|
+
n = int(raw)
|
|
436
|
+
except ValueError:
|
|
437
|
+
return 3
|
|
438
|
+
return n if n >= 0 else 3
|
|
439
|
+
|
|
440
|
+
def _path(self):
|
|
441
|
+
return get_loki_dir() / "health.json"
|
|
442
|
+
|
|
443
|
+
def _load(self):
|
|
444
|
+
try:
|
|
445
|
+
data = json.loads(self._path().read_text(encoding="utf-8"))
|
|
446
|
+
except (OSError, ValueError):
|
|
447
|
+
return {}
|
|
448
|
+
return data if isinstance(data, dict) else {}
|
|
449
|
+
|
|
450
|
+
def _save(self, data):
|
|
451
|
+
path = self._path()
|
|
452
|
+
tmp = path.with_name("health.json.%d.tmp" % os.getpid())
|
|
453
|
+
tmp.write_text(json.dumps(data, indent=1), encoding="utf-8")
|
|
454
|
+
os.replace(tmp, path)
|
|
455
|
+
|
|
456
|
+
@staticmethod
|
|
457
|
+
def _entry(data, source):
|
|
458
|
+
e = data.get(source)
|
|
459
|
+
if not isinstance(e, dict):
|
|
460
|
+
e = {}
|
|
461
|
+
e.setdefault("runs", 0)
|
|
462
|
+
e.setdefault("passed", 0)
|
|
463
|
+
e.setdefault("consecutive_failures", 0)
|
|
464
|
+
e.setdefault("paused", False)
|
|
465
|
+
e.setdefault("paused_at", "")
|
|
466
|
+
if not isinstance(e.get("durations_s"), list):
|
|
467
|
+
e["durations_s"] = []
|
|
468
|
+
data[source] = e
|
|
469
|
+
return e
|
|
470
|
+
|
|
471
|
+
def record(self, source, returncode, elapsed_s):
|
|
472
|
+
if returncode in (130, 143):
|
|
473
|
+
return
|
|
474
|
+
newly_paused = 0
|
|
475
|
+
with self._lock:
|
|
476
|
+
data = self._load()
|
|
477
|
+
e = self._entry(data, source)
|
|
478
|
+
e["runs"] += 1
|
|
479
|
+
if returncode == 0:
|
|
480
|
+
e["passed"] += 1
|
|
481
|
+
e["consecutive_failures"] = 0
|
|
482
|
+
e["durations_s"] = (e["durations_s"] + [round(elapsed_s, 1)])[-DURATIONS_KEPT:]
|
|
483
|
+
else:
|
|
484
|
+
e["consecutive_failures"] += 1
|
|
485
|
+
limit = self.pause_after()
|
|
486
|
+
if (limit > 0 and e["consecutive_failures"] >= limit
|
|
487
|
+
and source not in AUTOPAUSE_EXEMPT and not e["paused"]):
|
|
488
|
+
e["paused"] = True
|
|
489
|
+
e["paused_at"] = _utc_now()
|
|
490
|
+
newly_paused = e["consecutive_failures"]
|
|
491
|
+
try:
|
|
492
|
+
self._save(data)
|
|
493
|
+
except OSError as err:
|
|
494
|
+
logging.error("trigger health write failed: %s", err)
|
|
495
|
+
if newly_paused:
|
|
496
|
+
log_event(source, "", "%d consecutive failed runs" % newly_paused, "auto-paused")
|
|
497
|
+
send_notification(
|
|
498
|
+
"Trigger source %s paused after %d consecutive failed runs; resume with: loki trigger resume %s"
|
|
499
|
+
% (source, newly_paused, source))
|
|
500
|
+
|
|
501
|
+
def is_paused(self, source):
|
|
502
|
+
with self._lock:
|
|
503
|
+
e = self._load().get(source)
|
|
504
|
+
return bool(isinstance(e, dict) and e.get("paused"))
|
|
505
|
+
|
|
506
|
+
def set_paused(self, source, flag):
|
|
507
|
+
with self._lock:
|
|
508
|
+
data = self._load()
|
|
509
|
+
e = self._entry(data, source)
|
|
510
|
+
e["paused"] = bool(flag)
|
|
511
|
+
e["paused_at"] = _utc_now() if flag else ""
|
|
512
|
+
if not flag:
|
|
513
|
+
e["consecutive_failures"] = 0
|
|
514
|
+
self._save(data)
|
|
515
|
+
|
|
516
|
+
def snapshot(self):
|
|
517
|
+
with self._lock:
|
|
518
|
+
return self._load()
|
|
519
|
+
|
|
520
|
+
def paused_sources(self):
|
|
521
|
+
return sorted(k for k, v in self.snapshot().items() if isinstance(v, dict) and v.get("paused"))
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
HEALTH = TriggerHealth()
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def health_lines():
|
|
528
|
+
"""One status line per source that has any record (the lane metric)."""
|
|
529
|
+
snap = HEALTH.snapshot()
|
|
530
|
+
lines = []
|
|
531
|
+
for src in ALL_SOURCES:
|
|
532
|
+
e = snap.get(src)
|
|
533
|
+
if not isinstance(e, dict) or not (e.get("runs") or e.get("paused")):
|
|
534
|
+
continue
|
|
535
|
+
d = sorted(x for x in e.get("durations_s", []) if isinstance(x, (int, float)))
|
|
536
|
+
median = ("%ds" % round(d[len(d) // 2] if len(d) % 2 else (d[len(d) // 2 - 1] + d[len(d) // 2]) / 2)) if d else "n/a"
|
|
537
|
+
state = "PAUSED since %s" % e.get("paused_at") if e.get("paused") else "active"
|
|
538
|
+
lines.append("%s: runs %d, passed %d (exit 0 = VERIFIED for start runs), consecutive failures %d, median %s trigger-to-exit, %s"
|
|
539
|
+
% (src, e.get("runs", 0), e.get("passed", 0), e.get("consecutive_failures", 0), median, state))
|
|
540
|
+
return lines
|
|
541
|
+
|
|
542
|
+
|
|
417
543
|
def validate_signature(secret, body, signature_header):
|
|
418
544
|
"""Validate GitHub HMAC-SHA256 webhook signature (constant-time).
|
|
419
545
|
|
|
@@ -471,7 +597,7 @@ def _remove_quietly(*paths):
|
|
|
471
597
|
pass
|
|
472
598
|
|
|
473
599
|
|
|
474
|
-
def _reap_child(proc):
|
|
600
|
+
def _reap_child(proc, on_exit=None):
|
|
475
601
|
"""Wait on a child process so it is always reaped (no zombie).
|
|
476
602
|
|
|
477
603
|
Used when the dispatch outlives our synchronous wait window: instead of
|
|
@@ -485,7 +611,12 @@ def _reap_child(proc):
|
|
|
485
611
|
try:
|
|
486
612
|
proc.wait()
|
|
487
613
|
except Exception:
|
|
488
|
-
|
|
614
|
+
return
|
|
615
|
+
if on_exit is not None:
|
|
616
|
+
try:
|
|
617
|
+
on_exit(proc.returncode)
|
|
618
|
+
except Exception as e:
|
|
619
|
+
logging.error("on_exit callback failed: %s", e)
|
|
489
620
|
|
|
490
621
|
|
|
491
622
|
# Outcome of a dispatch, as distinct from "did it launch".
|
|
@@ -527,13 +658,24 @@ def _scoped_child_env(extra_env=None):
|
|
|
527
658
|
return env
|
|
528
659
|
|
|
529
660
|
|
|
530
|
-
|
|
661
|
+
# Commands whose exit code feeds the per-source trigger health counters.
|
|
662
|
+
RUN_COMMANDS = ("start", "verify-pr")
|
|
663
|
+
|
|
664
|
+
# Set by dispatch_event on the worker thread: the trigger source of the event
|
|
665
|
+
# being handled, so run_loki_outcome can attribute the run's exit code to it.
|
|
666
|
+
_DISPATCH = threading.local()
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None, cwd=None, on_exit=None):
|
|
531
670
|
"""Run a loki command and report WHICH of the three outcomes occurred.
|
|
532
671
|
|
|
533
672
|
Returns one of OUTCOME_PASSED / OUTCOME_FAILED / OUTCOME_UNKNOWN. This is
|
|
534
673
|
the honest version of run_loki_command, which answers only "did it launch".
|
|
535
674
|
When stdout_sink is a list, the decoded stdout is appended to it on completion.
|
|
536
675
|
extra_env adds child env vars (e.g. LOKI_STEER_SOURCE) on top of the scoped env.
|
|
676
|
+
on_exit(returncode) is called once the child ends, also from the reaper thread
|
|
677
|
+
for a detached run; it gets None when the launch itself failed. A start or
|
|
678
|
+
verify-pr run launched for a trigger source is also recorded in HEALTH.
|
|
537
679
|
"""
|
|
538
680
|
cmd = ["loki"] + args
|
|
539
681
|
if dry_run:
|
|
@@ -543,6 +685,15 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
|
|
|
543
685
|
# Child output goes to per-run files, never a pipe: past the wait window
|
|
544
686
|
# nothing reads a pipe, so a child writing more than the ~64KB pipe buffer
|
|
545
687
|
# would block forever. The files double as the run log for a detached child.
|
|
688
|
+
source = getattr(_DISPATCH, "source", None)
|
|
689
|
+
if source and args and args[0] in RUN_COMMANDS:
|
|
690
|
+
t0 = time.monotonic()
|
|
691
|
+
user_cb = on_exit
|
|
692
|
+
|
|
693
|
+
def on_exit(rc, _src=source, _t0=t0, _cb=user_cb):
|
|
694
|
+
HEALTH.record(_src, rc, time.monotonic() - _t0)
|
|
695
|
+
if _cb is not None:
|
|
696
|
+
_cb(rc)
|
|
546
697
|
out_path, err_path = _dispatch_log_paths()
|
|
547
698
|
try:
|
|
548
699
|
with open(out_path, "wb") as out_f, open(err_path, "wb") as err_f:
|
|
@@ -551,10 +702,13 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
|
|
|
551
702
|
stdout=out_f,
|
|
552
703
|
stderr=err_f,
|
|
553
704
|
env=_scoped_child_env(extra_env),
|
|
705
|
+
cwd=cwd,
|
|
554
706
|
)
|
|
555
707
|
except (FileNotFoundError, OSError) as e:
|
|
556
708
|
logging.error("Failed to launch %s: %s", " ".join(cmd), e)
|
|
557
709
|
_remove_quietly(out_path, err_path)
|
|
710
|
+
if on_exit is not None:
|
|
711
|
+
on_exit(None)
|
|
558
712
|
return OUTCOME_FAILED
|
|
559
713
|
|
|
560
714
|
try:
|
|
@@ -571,7 +725,7 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
|
|
|
571
725
|
)
|
|
572
726
|
threading.Thread(
|
|
573
727
|
target=_reap_child,
|
|
574
|
-
args=(proc,),
|
|
728
|
+
args=(proc, on_exit),
|
|
575
729
|
name="loki-trigger-reaper-%d" % proc.pid,
|
|
576
730
|
daemon=True,
|
|
577
731
|
).start()
|
|
@@ -582,6 +736,11 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
|
|
|
582
736
|
_remove_quietly(out_path, err_path)
|
|
583
737
|
if stdout_sink is not None:
|
|
584
738
|
stdout_sink.append(stdout.decode("utf-8", errors="replace"))
|
|
739
|
+
if on_exit is not None:
|
|
740
|
+
try:
|
|
741
|
+
on_exit(proc.returncode)
|
|
742
|
+
except Exception as e:
|
|
743
|
+
logging.error("on_exit callback failed: %s", e)
|
|
585
744
|
|
|
586
745
|
if proc.returncode == 0:
|
|
587
746
|
logging.info("Dispatch pid=%d completed (exit 0)", proc.pid)
|
|
@@ -599,7 +758,7 @@ def run_loki_outcome(args, dry_run=False, stdout_sink=None, extra_env=None):
|
|
|
599
758
|
return OUTCOME_FAILED
|
|
600
759
|
|
|
601
760
|
|
|
602
|
-
def run_loki_command(args, dry_run=False, stdout_sink=None):
|
|
761
|
+
def run_loki_command(args, dry_run=False, stdout_sink=None, cwd=None, on_exit=None):
|
|
603
762
|
"""Run a loki command synchronously and reap it; or print it if dry_run.
|
|
604
763
|
|
|
605
764
|
Returns True if the command was launched and exited 0 (or backgrounded
|
|
@@ -615,7 +774,7 @@ def run_loki_command(args, dry_run=False, stdout_sink=None):
|
|
|
615
774
|
passed wants run_loki_outcome instead. Behaviour is unchanged: a detached
|
|
616
775
|
dispatch (UNKNOWN) still reads as True here, exactly as before.
|
|
617
776
|
"""
|
|
618
|
-
return run_loki_outcome(args, dry_run=dry_run, stdout_sink=stdout_sink) in (
|
|
777
|
+
return run_loki_outcome(args, dry_run=dry_run, stdout_sink=stdout_sink, cwd=cwd, on_exit=on_exit) in (
|
|
619
778
|
OUTCOME_PASSED, OUTCOME_UNKNOWN,
|
|
620
779
|
)
|
|
621
780
|
|
|
@@ -874,8 +1033,33 @@ def handle_pull_request_event(payload, dry_run=False):
|
|
|
874
1033
|
return summary, status
|
|
875
1034
|
|
|
876
1035
|
|
|
1036
|
+
def build_ci_brief(wf, repo):
|
|
1037
|
+
"""One-line, control-char-free brief for a red default-branch CI run, or "" if unusable."""
|
|
1038
|
+
run_id = wf.get("id")
|
|
1039
|
+
if not isinstance(run_id, int) or isinstance(run_id, bool) or run_id <= 0:
|
|
1040
|
+
return ""
|
|
1041
|
+
sha = str(wf.get("head_sha") or "")
|
|
1042
|
+
sha7 = sha[:7] if re.fullmatch(r"[0-9a-fA-F]{7,40}", sha) else "unknown"
|
|
1043
|
+
url = str(wf.get("html_url") or "")
|
|
1044
|
+
where = "run %d" % run_id
|
|
1045
|
+
if url.startswith("https://github.com/"):
|
|
1046
|
+
where += ", %s" % _clip(url, 200)
|
|
1047
|
+
brief = (
|
|
1048
|
+
"CI is red on the default branch of %s: workflow '%s' (%s) failed at commit %s on %s. "
|
|
1049
|
+
"Read the failure with `gh run view %d --repo %s --log-failed`, reproduce it locally, "
|
|
1050
|
+
"fix the root cause with the smallest change, and keep every existing test passing. "
|
|
1051
|
+
"Do not skip, disable or delete tests or CI steps."
|
|
1052
|
+
% (repo, _clip(wf.get("name")), where, sha7, _clip(wf.get("head_branch")), run_id, repo))
|
|
1053
|
+
brief = " ".join(CONTROL_CHARS_RE.sub(" ", brief).split())
|
|
1054
|
+
return brief.encode("utf-8")[:SENTRY_BRIEF_MAX].decode("utf-8", errors="ignore")
|
|
1055
|
+
|
|
1056
|
+
|
|
877
1057
|
def handle_workflow_run_event(payload, dry_run=False):
|
|
878
|
-
"""Handle workflow_run
|
|
1058
|
+
"""Handle workflow_run: red default-branch CI -> loki start --brief <CI brief> --pr.
|
|
1059
|
+
|
|
1060
|
+
Only the default branch: a red PR branch is `loki merge watch`'s job, and Loki's
|
|
1061
|
+
own loki/* branches must never retrigger themselves.
|
|
1062
|
+
"""
|
|
879
1063
|
action = payload.get("action", "")
|
|
880
1064
|
if action != "completed":
|
|
881
1065
|
return None, "skipped (action=%s)" % action
|
|
@@ -883,14 +1067,22 @@ def handle_workflow_run_event(payload, dry_run=False):
|
|
|
883
1067
|
conclusion = wf.get("conclusion", "")
|
|
884
1068
|
if conclusion != "failure":
|
|
885
1069
|
return None, "skipped (conclusion=%s)" % conclusion
|
|
886
|
-
wf_name = wf.get("name", "unknown")
|
|
887
1070
|
repo = payload.get("repository", {})
|
|
888
1071
|
repo_full_name = repo.get("full_name", "")
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
1072
|
+
if not valid_repo_full_name(repo_full_name):
|
|
1073
|
+
return None, "rejected (invalid repository name)"
|
|
1074
|
+
default = repo.get("default_branch", "")
|
|
1075
|
+
if not default:
|
|
1076
|
+
return None, "skipped (no default_branch in payload)"
|
|
1077
|
+
branch = str(wf.get("head_branch") or "")
|
|
1078
|
+
if branch != default or branch.startswith("loki/"):
|
|
1079
|
+
return None, ("skipped (failure on branch %s, not the default branch; "
|
|
1080
|
+
"use loki merge watch for PR branches)" % _clip(branch))
|
|
1081
|
+
brief = build_ci_brief(wf, repo_full_name)
|
|
1082
|
+
if not brief or not valid_spec(brief):
|
|
1083
|
+
return None, "rejected (invalid brief)"
|
|
1084
|
+
summary = "workflow '%s' failed in %s" % (_clip(wf.get("name", "unknown")), repo_full_name)
|
|
1085
|
+
success = run_loki_command(brief_start_args(brief), dry_run=dry_run)
|
|
894
1086
|
status = "fired" if success else "error"
|
|
895
1087
|
if success:
|
|
896
1088
|
send_notification("Trigger fired: CI failure - %s" % summary)
|
|
@@ -1398,17 +1590,126 @@ def handle_sentry_event(payload, dry_run=False):
|
|
|
1398
1590
|
return summary, status
|
|
1399
1591
|
|
|
1400
1592
|
|
|
1593
|
+
REVIEW_REF_RE = re.compile(r"^[A-Za-z0-9._][A-Za-z0-9._/-]{0,199}$")
|
|
1594
|
+
FULL_SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$")
|
|
1595
|
+
REVIEW_CLONE_TIMEOUT = 120
|
|
1596
|
+
|
|
1597
|
+
|
|
1598
|
+
def _review_root():
|
|
1599
|
+
return (get_loki_dir() / "review").resolve()
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
def _remove_review_clone(path):
|
|
1603
|
+
"""Delete a review clone, only when it sits under the review root."""
|
|
1604
|
+
try:
|
|
1605
|
+
real = Path(path).resolve()
|
|
1606
|
+
if real.parent == _review_root():
|
|
1607
|
+
shutil.rmtree(real, ignore_errors=True)
|
|
1608
|
+
except OSError as e:
|
|
1609
|
+
logging.error("review clone cleanup failed: %s", e)
|
|
1610
|
+
|
|
1611
|
+
|
|
1612
|
+
def handle_pull_request_review_event(payload, dry_run=False):
|
|
1613
|
+
"""A trusted 'changes requested' review runs `loki verify-pr respond` on a fresh clone of the PR branch."""
|
|
1614
|
+
action = payload.get("action", "")
|
|
1615
|
+
if action != "submitted":
|
|
1616
|
+
return None, "skipped (action=%s)" % action
|
|
1617
|
+
review = payload.get("review") or {}
|
|
1618
|
+
state = str(review.get("state", "")).lower()
|
|
1619
|
+
if state != "changes_requested":
|
|
1620
|
+
return None, "skipped (review state=%s)" % _clip(state, 30)
|
|
1621
|
+
if review.get("author_association") not in TRUSTED_ASSOCIATIONS:
|
|
1622
|
+
return None, "skipped (review author not authorized)"
|
|
1623
|
+
user = review.get("user") or {}
|
|
1624
|
+
if user.get("type") == "Bot" or str(user.get("login", "")).endswith("[bot]"):
|
|
1625
|
+
return None, "skipped (bot review)"
|
|
1626
|
+
if LOKI_COMMENT_MARKER in str(review.get("body") or ""):
|
|
1627
|
+
return None, "skipped (loki-authored review)"
|
|
1628
|
+
pr = payload.get("pull_request") or {}
|
|
1629
|
+
repo_full_name = (payload.get("repository") or {}).get("full_name", "")
|
|
1630
|
+
if not valid_repo_full_name(repo_full_name):
|
|
1631
|
+
return None, "rejected (invalid repository full_name)"
|
|
1632
|
+
number = pr.get("number")
|
|
1633
|
+
if not valid_issue_number(number):
|
|
1634
|
+
return None, "rejected (invalid pull request number)"
|
|
1635
|
+
if pr.get("state") != "open":
|
|
1636
|
+
return None, "skipped (pull request not open)"
|
|
1637
|
+
head = pr.get("head") or {}
|
|
1638
|
+
if (head.get("repo") or {}).get("full_name") != repo_full_name:
|
|
1639
|
+
return None, "skipped (pull request head is a fork)"
|
|
1640
|
+
ref = str(head.get("ref") or "")
|
|
1641
|
+
if not REVIEW_REF_RE.match(ref) or ".." in ref:
|
|
1642
|
+
return None, "rejected (invalid head ref)"
|
|
1643
|
+
sha = str(head.get("sha") or "")
|
|
1644
|
+
if not FULL_SHA_RE.match(sha):
|
|
1645
|
+
return None, "rejected (invalid head sha)"
|
|
1646
|
+
summary = "review changes requested on %s#%s" % (repo_full_name, number)
|
|
1647
|
+
if dry_run:
|
|
1648
|
+
run_loki_command(["verify-pr", "respond", "%s#%s" % (repo_full_name, number)], dry_run=True)
|
|
1649
|
+
return summary, "fired"
|
|
1650
|
+
|
|
1651
|
+
env = _scoped_child_env()
|
|
1652
|
+
try:
|
|
1653
|
+
origin = subprocess.run(["git", "remote", "get-url", "origin"], capture_output=True, text=True,
|
|
1654
|
+
timeout=30, env=env).stdout.strip()
|
|
1655
|
+
except (OSError, subprocess.SubprocessError):
|
|
1656
|
+
origin = ""
|
|
1657
|
+
if not origin:
|
|
1658
|
+
return summary, "error"
|
|
1659
|
+
root = _review_root()
|
|
1660
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
1661
|
+
clone = root / ("pr-%s-%s" % (number, secrets.token_hex(4)))
|
|
1662
|
+
try:
|
|
1663
|
+
subprocess.run(["git", "clone", "--quiet", "--depth", "50", "--single-branch", "--branch", ref,
|
|
1664
|
+
"--", origin, str(clone)], capture_output=True, timeout=REVIEW_CLONE_TIMEOUT,
|
|
1665
|
+
env=env, check=True)
|
|
1666
|
+
got = subprocess.run(["git", "-C", str(clone), "rev-parse", "HEAD"], capture_output=True,
|
|
1667
|
+
text=True, timeout=30, env=env).stdout.strip()
|
|
1668
|
+
except (OSError, subprocess.SubprocessError) as e:
|
|
1669
|
+
logging.error("review clone failed: %s", e)
|
|
1670
|
+
_remove_review_clone(clone)
|
|
1671
|
+
HEALTH.record("review", None, 0.0)
|
|
1672
|
+
return summary, "error"
|
|
1673
|
+
if got.lower() != sha.lower():
|
|
1674
|
+
_remove_review_clone(clone)
|
|
1675
|
+
return None, "skipped (PR head moved)"
|
|
1676
|
+
success = run_loki_command(["verify-pr", "respond", "%s#%s" % (repo_full_name, number)],
|
|
1677
|
+
cwd=str(clone), on_exit=lambda rc: _remove_review_clone(clone))
|
|
1678
|
+
if success:
|
|
1679
|
+
send_notification("Trigger fired: %s" % summary)
|
|
1680
|
+
else:
|
|
1681
|
+
_remove_review_clone(clone)
|
|
1682
|
+
return summary, "fired" if success else "error"
|
|
1683
|
+
|
|
1684
|
+
|
|
1401
1685
|
# Map event name -> handler. Keeps do_POST routing declarative. JOB_EVENT rides
|
|
1402
1686
|
# the same table (and therefore the same bounded queue and worker pool) but is
|
|
1403
1687
|
# explicitly refused on the /webhook path.
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
"
|
|
1407
|
-
"
|
|
1408
|
-
"
|
|
1409
|
-
|
|
1410
|
-
|
|
1688
|
+
# event -> (source name, handler). A None source is steer-only: never counted or paused.
|
|
1689
|
+
TRIGGERS = {
|
|
1690
|
+
"issues": ("issues", handle_issues_event),
|
|
1691
|
+
"issue_comment": (None, handle_issue_comment_event),
|
|
1692
|
+
"pull_request": ("pull_request", handle_pull_request_event),
|
|
1693
|
+
"workflow_run": ("ci", handle_workflow_run_event),
|
|
1694
|
+
"pull_request_review": ("review", handle_pull_request_review_event),
|
|
1695
|
+
JOB_EVENT: ("jobs", handle_job_event),
|
|
1696
|
+
SENTRY_EVENT: ("sentry", handle_sentry_event),
|
|
1411
1697
|
}
|
|
1698
|
+
EVENT_HANDLERS = {k: v[1] for k, v in TRIGGERS.items()}
|
|
1699
|
+
|
|
1700
|
+
# Sources that share the job queue; _enqueue_job stamps the job with its source.
|
|
1701
|
+
JOB_SOURCES = ("jobs", "slack", "linear", "jira", "teams")
|
|
1702
|
+
ALL_SOURCES = tuple(sorted({v[0] for v in TRIGGERS.values() if v[0]} | set(JOB_SOURCES)))
|
|
1703
|
+
# A direct /jobs caller already gets a per-job failed status and decides for itself.
|
|
1704
|
+
AUTOPAUSE_EXEMPT = ("jobs",)
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
def trigger_source(event_type, payload):
|
|
1708
|
+
if event_type == JOB_EVENT:
|
|
1709
|
+
src = payload.get("source") if isinstance(payload, dict) else None
|
|
1710
|
+
return src if src in JOB_SOURCES else "jobs"
|
|
1711
|
+
entry = TRIGGERS.get(event_type)
|
|
1712
|
+
return entry[0] if entry else None
|
|
1412
1713
|
|
|
1413
1714
|
|
|
1414
1715
|
def dispatch_event(event_type, payload, dry_run=False):
|
|
@@ -1423,11 +1724,20 @@ def dispatch_event(event_type, payload, dry_run=False):
|
|
|
1423
1724
|
status = "unsupported event: %s" % event_type
|
|
1424
1725
|
log_event(event_type, action, "", status)
|
|
1425
1726
|
return None, status
|
|
1727
|
+
source = trigger_source(event_type, payload)
|
|
1728
|
+
if source and HEALTH.is_paused(source):
|
|
1729
|
+
status = "paused (source %s; loki trigger resume %s)" % (source, source)
|
|
1730
|
+
log_event(event_type, action, "", status)
|
|
1731
|
+
# A queued job needs a terminal status so its submitter does not poll forever.
|
|
1732
|
+
return (status, JOB_STATUS_FAILED) if event_type == JOB_EVENT else (None, "paused")
|
|
1733
|
+
_DISPATCH.source = source
|
|
1426
1734
|
try:
|
|
1427
1735
|
summary, status = handler(payload, dry_run=dry_run)
|
|
1428
1736
|
except Exception as e: # defensive: never let a worker die on bad input
|
|
1429
1737
|
logging.exception("Handler for %s raised: %s", event_type, e)
|
|
1430
1738
|
summary, status = None, "error"
|
|
1739
|
+
finally:
|
|
1740
|
+
_DISPATCH.source = None
|
|
1431
1741
|
log_event(event_type, action, summary or "", status)
|
|
1432
1742
|
return summary, status
|
|
1433
1743
|
|
|
@@ -1521,9 +1831,24 @@ class Dispatcher:
|
|
|
1521
1831
|
"""True if the same repo#N + action + head sha was accepted and its run is live.
|
|
1522
1832
|
|
|
1523
1833
|
Catches a redelivery under a new delivery id or after a restart. Once the
|
|
1524
|
-
run is no longer live the same key may dispatch again.
|
|
1525
|
-
|
|
1834
|
+
run is no longer live the same key may dispatch again. A red workflow_run
|
|
1835
|
+
has no issue ref: one repair run per repo and head sha, across workflows
|
|
1836
|
+
and redeliveries, so its key stays remembered.
|
|
1526
1837
|
"""
|
|
1838
|
+
if event_type == "workflow_run":
|
|
1839
|
+
repo = (payload.get("repository") or {}).get("full_name", "")
|
|
1840
|
+
wf = payload.get("workflow_run") or {}
|
|
1841
|
+
sha = wf.get("head_sha", "")
|
|
1842
|
+
# Only a red completed run takes the key: a green run of the same sha must not mask it.
|
|
1843
|
+
if (payload.get("action") != "completed" or wf.get("conclusion") != "failure"
|
|
1844
|
+
or not repo or not sha):
|
|
1845
|
+
return False
|
|
1846
|
+
key = "intake:workflow_run:%s@%s" % (repo, sha)
|
|
1847
|
+
with self._dedup_lock:
|
|
1848
|
+
if key in self._seen_deliveries:
|
|
1849
|
+
return True
|
|
1850
|
+
self._remember(key)
|
|
1851
|
+
return False
|
|
1527
1852
|
if event_type not in ("issues", "pull_request"):
|
|
1528
1853
|
return False
|
|
1529
1854
|
number = (payload.get("issue") or payload.get("pull_request") or {}).get("number")
|
|
@@ -1816,7 +2141,8 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
1816
2141
|
"status": "running",
|
|
1817
2142
|
"dry_run": self.dry_run,
|
|
1818
2143
|
"port": config.get("port", 7373),
|
|
1819
|
-
"
|
|
2144
|
+
"triggers": HEALTH.snapshot(),
|
|
2145
|
+
"paused": HEALTH.paused_sources(),
|
|
1820
2146
|
"secret_configured": bool(self.secret),
|
|
1821
2147
|
"api_token_configured": bool(self.api_token),
|
|
1822
2148
|
})
|
|
@@ -2097,14 +2423,14 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
2097
2423
|
return
|
|
2098
2424
|
self._send_json(202, {"id": job_id, "status": "queued"})
|
|
2099
2425
|
|
|
2100
|
-
def _enqueue_job(self, spec, reply_target=None, pr=False, unit_job=None):
|
|
2426
|
+
def _enqueue_job(self, spec, reply_target=None, pr=False, unit_job=None, source="jobs"):
|
|
2101
2427
|
"""Record and enqueue one validated job; shared by /jobs, /slack, /linear.
|
|
2102
2428
|
|
|
2103
2429
|
Returns the job id, or None after sending 429 when the queue is full.
|
|
2104
2430
|
reply_target must come from an already-verified request.
|
|
2105
2431
|
"""
|
|
2106
2432
|
job_id = secrets.token_urlsafe(12)
|
|
2107
|
-
job = {"spec": spec, "job_id": job_id}
|
|
2433
|
+
job = {"spec": spec, "job_id": job_id, "source": source}
|
|
2108
2434
|
if pr:
|
|
2109
2435
|
job["pr"] = True
|
|
2110
2436
|
if unit_job is not None:
|
|
@@ -2183,7 +2509,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
2183
2509
|
target = {"kind": "slack"}
|
|
2184
2510
|
if response_url.startswith(SLACK_REPLY_PREFIX):
|
|
2185
2511
|
target["response_url"] = response_url
|
|
2186
|
-
job_id = self._enqueue_job(text.strip(), target)
|
|
2512
|
+
job_id = self._enqueue_job(text.strip(), target, source="slack")
|
|
2187
2513
|
if job_id is None:
|
|
2188
2514
|
return
|
|
2189
2515
|
self._send_json(200, {"response_type": "ephemeral",
|
|
@@ -2240,7 +2566,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
2240
2566
|
issue_id = data.get("id")
|
|
2241
2567
|
target = {"kind": "linear",
|
|
2242
2568
|
"issue_id": issue_id if isinstance(issue_id, str) else ""}
|
|
2243
|
-
job_id = self._enqueue_job(spec, target)
|
|
2569
|
+
job_id = self._enqueue_job(spec, target, source="linear")
|
|
2244
2570
|
if job_id is None:
|
|
2245
2571
|
return
|
|
2246
2572
|
self._send_json(202, {"id": job_id, "status": "queued"})
|
|
@@ -2298,7 +2624,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
2298
2624
|
if dup_id or dup_body:
|
|
2299
2625
|
self._send_json(200, {"status": "duplicate"})
|
|
2300
2626
|
return
|
|
2301
|
-
job_id = self._enqueue_job(brief, {"kind": "jira", "key": key}, pr=True)
|
|
2627
|
+
job_id = self._enqueue_job(brief, {"kind": "jira", "key": key}, pr=True, source="jira")
|
|
2302
2628
|
if job_id is None:
|
|
2303
2629
|
return
|
|
2304
2630
|
self._send_json(202, {"id": job_id, "status": "queued"})
|
|
@@ -2339,7 +2665,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
2339
2665
|
if self.dispatcher.seen_delivery(teams_dedup_key(activity)):
|
|
2340
2666
|
self._send_json(200, {"type": "message", "text": "Duplicate, already queued."})
|
|
2341
2667
|
return
|
|
2342
|
-
job_id = self._enqueue_job(brief, pr=True)
|
|
2668
|
+
job_id = self._enqueue_job(brief, pr=True, source="teams")
|
|
2343
2669
|
if job_id is None:
|
|
2344
2670
|
return
|
|
2345
2671
|
self._send_json(200, {"type": "message", "text": "Queued loki job %s" % job_id})
|
|
@@ -2544,8 +2870,23 @@ def main():
|
|
|
2544
2870
|
parser.add_argument("--dry-run", action="store_true", help="Preview triggers without running loki")
|
|
2545
2871
|
parser.add_argument("--workers", type=int, default=None, help="Worker threads draining the dispatch queue")
|
|
2546
2872
|
parser.add_argument("--queue-size", type=int, default=None, help="Max in-flight queued dispatches")
|
|
2873
|
+
ctl = parser.add_mutually_exclusive_group()
|
|
2874
|
+
ctl.add_argument("--health", action="store_true", help="Print per-source run counts and pause state, then exit")
|
|
2875
|
+
ctl.add_argument("--resume", metavar="SOURCE", help="Resume a paused trigger source, then exit")
|
|
2547
2876
|
args = parser.parse_args()
|
|
2548
2877
|
|
|
2878
|
+
if args.health:
|
|
2879
|
+
lines = health_lines()
|
|
2880
|
+
print("\n".join(lines) if lines else "No triggered runs recorded yet.")
|
|
2881
|
+
return 0
|
|
2882
|
+
if args.resume is not None:
|
|
2883
|
+
if args.resume not in ALL_SOURCES:
|
|
2884
|
+
print("unknown trigger source: %s (known: %s)" % (args.resume, ", ".join(ALL_SOURCES)))
|
|
2885
|
+
return 2
|
|
2886
|
+
HEALTH.set_paused(args.resume, False)
|
|
2887
|
+
print("%s: resumed" % args.resume)
|
|
2888
|
+
return 0
|
|
2889
|
+
|
|
2549
2890
|
logging.basicConfig(
|
|
2550
2891
|
level=logging.INFO,
|
|
2551
2892
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
@@ -2712,7 +3053,7 @@ if __name__ == "__main__":
|
|
|
2712
3053
|
if hasattr(signal, "SIGPIPE"):
|
|
2713
3054
|
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
|
2714
3055
|
|
|
2715
|
-
main()
|
|
3056
|
+
rc = main()
|
|
2716
3057
|
|
|
2717
3058
|
# ponytail: flush stdout/stderr inside try/except BrokenPipeError before
|
|
2718
3059
|
# exit, then call os._exit() to avoid daemon thread cleanup during
|
|
@@ -2725,4 +3066,4 @@ if __name__ == "__main__":
|
|
|
2725
3066
|
sys.stderr.flush()
|
|
2726
3067
|
except BrokenPipeError:
|
|
2727
3068
|
pass
|
|
2728
|
-
os._exit(0)
|
|
3069
|
+
os._exit(rc if isinstance(rc, int) else 0)
|