loki-mode 7.78.0 → 7.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/lib/checkpoint_sync.py +189 -0
- package/autonomy/lib/config-map.sh +955 -0
- package/autonomy/loki +273 -8
- package/autonomy/run.sh +230 -195
- package/autonomy/sandbox.sh +98 -0
- package/autonomy/trigger-server.py +419 -43
- package/dashboard/__init__.py +1 -1
- package/dashboard/registry.py +212 -0
- package/dashboard/server.py +299 -7
- package/dashboard/static/index.html +383 -150
- package/docs/CONFIG-FILE-PLAN.md +459 -0
- package/docs/ENTERPRISE-IDENTITY-ROADMAP.md +206 -0
- package/docs/INSTALLATION.md +2 -2
- package/loki-ts/dist/loki.js +2 -2
- package/lokistore/__init__.py +65 -0
- package/lokistore/base.py +172 -0
- package/lokistore/cloud.py +305 -0
- package/lokistore/factory.py +187 -0
- package/lokistore/local.py +219 -0
- package/mcp/__init__.py +1 -1
- package/memory/retrieval.py +147 -0
- package/memory/tree_index.py +499 -0
- package/memory/tree_search.py +305 -0
- package/package.json +2 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
|
@@ -2,20 +2,33 @@
|
|
|
2
2
|
"""
|
|
3
3
|
trigger-server.py - GitHub webhook receiver for loki-mode event-driven execution.
|
|
4
4
|
|
|
5
|
-
Listens for GitHub webhook events and automatically runs `loki
|
|
6
|
-
Supports signature validation,
|
|
5
|
+
Listens for GitHub webhook events and automatically runs `loki start` in
|
|
6
|
+
response. Supports constant-time HMAC-SHA256 signature validation, a bounded
|
|
7
|
+
worker queue so a webhook storm cannot fork unbounded builds, child-process
|
|
8
|
+
reaping (no zombies), and event logging.
|
|
9
|
+
|
|
10
|
+
Security note: a webhook secret is REQUIRED. If no secret is configured the
|
|
11
|
+
server still starts (so /health and /status stay available for operators), but
|
|
12
|
+
every webhook POST is rejected with 503 and an audit log line. The server never
|
|
13
|
+
silently accepts unauthenticated webhooks.
|
|
7
14
|
|
|
8
15
|
Usage:
|
|
9
16
|
python3 autonomy/trigger-server.py [--port PORT] [--secret SECRET] [--dry-run]
|
|
17
|
+
[--workers N] [--queue-size N]
|
|
10
18
|
"""
|
|
11
19
|
|
|
12
20
|
import argparse
|
|
21
|
+
import collections
|
|
13
22
|
import hashlib
|
|
14
23
|
import hmac
|
|
15
24
|
import http.server
|
|
16
25
|
import json
|
|
17
26
|
import logging
|
|
18
27
|
import os
|
|
28
|
+
import queue
|
|
29
|
+
import re
|
|
30
|
+
import socket
|
|
31
|
+
import socketserver
|
|
19
32
|
import subprocess
|
|
20
33
|
import sys
|
|
21
34
|
import threading
|
|
@@ -24,6 +37,53 @@ from datetime import datetime
|
|
|
24
37
|
from pathlib import Path
|
|
25
38
|
|
|
26
39
|
|
|
40
|
+
# A GitHub repository full_name is "owner/repo". Both segments are restricted to
|
|
41
|
+
# the characters GitHub itself allows (letters, digits, dot, underscore, dash).
|
|
42
|
+
# We validate against this before ever placing a webhook-supplied value into the
|
|
43
|
+
# `loki start <ref>` argv. Without this guard a payload whose repository
|
|
44
|
+
# full_name begins with "--" (e.g. "--config=/etc/x") would inject a CLI FLAG
|
|
45
|
+
# into `loki start` post-HMAC, breaking the "a webhook can only start a build
|
|
46
|
+
# for ref X" boundary. The ref we build ("owner/repo#N") therefore can never
|
|
47
|
+
# begin with a dash, so it can never be parsed as an option.
|
|
48
|
+
REPO_FULL_NAME_RE = re.compile(r"^[A-Za-z0-9._][A-Za-z0-9._-]*/[A-Za-z0-9._-]+$")
|
|
49
|
+
|
|
50
|
+
# Bound the body read so a slow/under-delivered POST cannot tie up a worker
|
|
51
|
+
# thread before authentication (slow-loris). A stalled client is dropped.
|
|
52
|
+
BODY_READ_TIMEOUT_SECONDS = 15
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def valid_repo_full_name(repo_full_name):
|
|
56
|
+
"""Return True if repo_full_name is a safe "owner/repo" string.
|
|
57
|
+
|
|
58
|
+
Rejects empty, malformed, or dash-leading values so a webhook-controlled
|
|
59
|
+
repository name can never be parsed as a `loki start` CLI flag.
|
|
60
|
+
"""
|
|
61
|
+
return bool(repo_full_name) and bool(REPO_FULL_NAME_RE.match(repo_full_name))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def valid_issue_number(number):
|
|
65
|
+
"""Return True if number is a positive integer (issue/PR number).
|
|
66
|
+
|
|
67
|
+
GitHub sends these as JSON integers. A non-int (or a string smuggled in via
|
|
68
|
+
a crafted payload) is rejected so only a clean integer reaches the ref.
|
|
69
|
+
"""
|
|
70
|
+
return isinstance(number, int) and not isinstance(number, bool) and number > 0
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# How long to wait for a dispatched `loki start` to finish before we stop
|
|
74
|
+
# waiting on it. The child is launched detached (--detach) so it backgrounds
|
|
75
|
+
# itself quickly; this bound only guards the worker thread against a wedged
|
|
76
|
+
# launch. The child keeps running independently after we stop waiting.
|
|
77
|
+
DISPATCH_WAIT_SECONDS = 30
|
|
78
|
+
|
|
79
|
+
# Defaults for the bounded worker queue.
|
|
80
|
+
DEFAULT_WORKERS = 4
|
|
81
|
+
DEFAULT_QUEUE_SIZE = 64
|
|
82
|
+
|
|
83
|
+
# How many recent GitHub delivery IDs to remember for idempotency.
|
|
84
|
+
DEFAULT_DEDUP_SIZE = 2048
|
|
85
|
+
|
|
86
|
+
|
|
27
87
|
def get_loki_dir():
|
|
28
88
|
"""Return .loki/triggers directory, creating it if needed."""
|
|
29
89
|
loki_dir = Path(".loki") / "triggers"
|
|
@@ -39,6 +99,8 @@ def load_config():
|
|
|
39
99
|
"secret": "",
|
|
40
100
|
"dry_run": False,
|
|
41
101
|
"enabled_events": ["issues", "pull_request", "workflow_run"],
|
|
102
|
+
"workers": DEFAULT_WORKERS,
|
|
103
|
+
"queue_size": DEFAULT_QUEUE_SIZE,
|
|
42
104
|
}
|
|
43
105
|
if config_path.exists():
|
|
44
106
|
try:
|
|
@@ -57,8 +119,13 @@ def save_config(config):
|
|
|
57
119
|
json.dump(config, f, indent=2)
|
|
58
120
|
|
|
59
121
|
|
|
122
|
+
# log_event writes append lines to a shared file from multiple worker threads,
|
|
123
|
+
# so serialize writes to avoid interleaved JSON lines.
|
|
124
|
+
_log_lock = threading.Lock()
|
|
125
|
+
|
|
126
|
+
|
|
60
127
|
def log_event(event_type, action, payload_summary, status):
|
|
61
|
-
"""Append event to .loki/triggers/events.log."""
|
|
128
|
+
"""Append event to .loki/triggers/events.log (thread-safe)."""
|
|
62
129
|
log_path = get_loki_dir() / "events.log"
|
|
63
130
|
timestamp = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
64
131
|
entry = {
|
|
@@ -68,24 +135,33 @@ def log_event(event_type, action, payload_summary, status):
|
|
|
68
135
|
"summary": payload_summary,
|
|
69
136
|
"status": status,
|
|
70
137
|
}
|
|
71
|
-
|
|
72
|
-
|
|
138
|
+
line = json.dumps(entry) + "\n"
|
|
139
|
+
with _log_lock:
|
|
140
|
+
with open(log_path, "a") as f:
|
|
141
|
+
f.write(line)
|
|
73
142
|
|
|
74
143
|
|
|
75
144
|
def validate_signature(secret, body, signature_header):
|
|
76
|
-
"""Validate GitHub HMAC-SHA256 webhook signature.
|
|
145
|
+
"""Validate GitHub HMAC-SHA256 webhook signature (constant-time).
|
|
146
|
+
|
|
147
|
+
Returns False when no secret is configured: an unauthenticated webhook is
|
|
148
|
+
never considered valid. The caller is responsible for refusing to dispatch
|
|
149
|
+
when no secret is set; this function only answers "is this request proven
|
|
150
|
+
to come from someone holding the secret?".
|
|
151
|
+
"""
|
|
77
152
|
if not secret:
|
|
78
|
-
return
|
|
153
|
+
return False
|
|
79
154
|
if not signature_header:
|
|
80
155
|
return False
|
|
81
156
|
expected = "sha256=" + hmac.new(
|
|
82
157
|
secret.encode("utf-8"), body, hashlib.sha256
|
|
83
158
|
).hexdigest()
|
|
159
|
+
# compare_digest is constant-time and tolerates unequal-length inputs.
|
|
84
160
|
return hmac.compare_digest(expected, signature_header)
|
|
85
161
|
|
|
86
162
|
|
|
87
163
|
def send_notification(message):
|
|
88
|
-
"""Send desktop notification via loki syslog."""
|
|
164
|
+
"""Send desktop notification via loki syslog. Reaps the child itself."""
|
|
89
165
|
try:
|
|
90
166
|
subprocess.run(
|
|
91
167
|
["loki", "syslog", message],
|
|
@@ -96,8 +172,31 @@ def send_notification(message):
|
|
|
96
172
|
pass
|
|
97
173
|
|
|
98
174
|
|
|
175
|
+
def _reap_child(proc):
|
|
176
|
+
"""Wait on a child process so it is always reaped (no zombie).
|
|
177
|
+
|
|
178
|
+
Used when the dispatch outlives our synchronous wait window: instead of
|
|
179
|
+
abandoning the child (which would leave a zombie until this process exits),
|
|
180
|
+
a one-shot daemon thread blocks on wait() until the child finishes. Errors
|
|
181
|
+
are swallowed because the only goal is to drain the child's exit status.
|
|
182
|
+
"""
|
|
183
|
+
try:
|
|
184
|
+
proc.wait()
|
|
185
|
+
except Exception:
|
|
186
|
+
pass
|
|
187
|
+
|
|
188
|
+
|
|
99
189
|
def run_loki_command(args, dry_run=False):
|
|
100
|
-
"""Run a loki command
|
|
190
|
+
"""Run a loki command synchronously and reap it; or print it if dry_run.
|
|
191
|
+
|
|
192
|
+
Returns True if the command was launched and exited 0 (or backgrounded
|
|
193
|
+
cleanly within the wait window), False if the launch failed or it exited
|
|
194
|
+
non-zero. The child is always waited on, so no zombies accumulate. stderr
|
|
195
|
+
is captured on failure so a broken dispatch is diagnosable.
|
|
196
|
+
|
|
197
|
+
This is invoked from worker threads, so blocking here does not block the
|
|
198
|
+
HTTP listener.
|
|
199
|
+
"""
|
|
101
200
|
cmd = ["loki"] + args
|
|
102
201
|
if dry_run:
|
|
103
202
|
logging.info("[DRY-RUN] Would run: %s", " ".join(cmd))
|
|
@@ -109,16 +208,51 @@ def run_loki_command(args, dry_run=False):
|
|
|
109
208
|
stdout=subprocess.PIPE,
|
|
110
209
|
stderr=subprocess.PIPE,
|
|
111
210
|
)
|
|
112
|
-
# Don't wait - detached execution
|
|
113
|
-
logging.info("Started process pid=%d", proc.pid)
|
|
114
|
-
return True
|
|
115
211
|
except (FileNotFoundError, OSError) as e:
|
|
116
|
-
logging.error("Failed to
|
|
212
|
+
logging.error("Failed to launch %s: %s", " ".join(cmd), e)
|
|
117
213
|
return False
|
|
118
214
|
|
|
215
|
+
try:
|
|
216
|
+
# Wait (and thereby reap) the child. A --detach launch returns quickly;
|
|
217
|
+
# this bound only guards against a wedged launch.
|
|
218
|
+
_, stderr = proc.communicate(timeout=DISPATCH_WAIT_SECONDS)
|
|
219
|
+
except subprocess.TimeoutExpired:
|
|
220
|
+
# The dispatch is still running past our wait window. We stop blocking
|
|
221
|
+
# the worker thread, but the child is NOT abandoned: a one-shot daemon
|
|
222
|
+
# reaper thread waits on it so it is always reaped (no zombie) while
|
|
223
|
+
# THIS process is alive, and the OS reparents it after we exit. This
|
|
224
|
+
# keeps the "always waited on, no zombies" guarantee honest.
|
|
225
|
+
logging.info(
|
|
226
|
+
"Dispatch pid=%d still running after %ds; reaping in background",
|
|
227
|
+
proc.pid,
|
|
228
|
+
DISPATCH_WAIT_SECONDS,
|
|
229
|
+
)
|
|
230
|
+
threading.Thread(
|
|
231
|
+
target=_reap_child,
|
|
232
|
+
args=(proc,),
|
|
233
|
+
name="loki-trigger-reaper-%d" % proc.pid,
|
|
234
|
+
daemon=True,
|
|
235
|
+
).start()
|
|
236
|
+
return True
|
|
237
|
+
|
|
238
|
+
if proc.returncode == 0:
|
|
239
|
+
logging.info("Dispatch pid=%d completed (exit 0)", proc.pid)
|
|
240
|
+
return True
|
|
241
|
+
|
|
242
|
+
stderr_text = ""
|
|
243
|
+
if stderr:
|
|
244
|
+
stderr_text = stderr.decode("utf-8", errors="replace").strip()
|
|
245
|
+
logging.error(
|
|
246
|
+
"Dispatch pid=%d exited %d: %s",
|
|
247
|
+
proc.pid,
|
|
248
|
+
proc.returncode,
|
|
249
|
+
stderr_text or "(no stderr)",
|
|
250
|
+
)
|
|
251
|
+
return False
|
|
252
|
+
|
|
119
253
|
|
|
120
254
|
def handle_issues_event(payload, dry_run=False):
|
|
121
|
-
"""Handle issues event: opened -> loki
|
|
255
|
+
"""Handle issues event: opened -> loki start <issue-ref> --pr --detach."""
|
|
122
256
|
action = payload.get("action", "")
|
|
123
257
|
if action != "opened":
|
|
124
258
|
return None, "skipped (action=%s)" % action
|
|
@@ -126,11 +260,19 @@ def handle_issues_event(payload, dry_run=False):
|
|
|
126
260
|
issue_number = issue.get("number")
|
|
127
261
|
repo = payload.get("repository", {})
|
|
128
262
|
repo_full_name = repo.get("full_name", "")
|
|
129
|
-
if
|
|
263
|
+
if issue_number is None:
|
|
130
264
|
return None, "skipped (no issue number)"
|
|
131
|
-
|
|
265
|
+
if not valid_issue_number(issue_number):
|
|
266
|
+
return None, "rejected (invalid issue number)"
|
|
267
|
+
# A repo full_name, when present, must be a clean "owner/repo". An invalid
|
|
268
|
+
# one (e.g. dash-leading) is rejected outright rather than silently dropped,
|
|
269
|
+
# so a flag-injection attempt is logged and never dispatched.
|
|
270
|
+
if repo_full_name and not valid_repo_full_name(repo_full_name):
|
|
271
|
+
return None, "rejected (invalid repository full_name)"
|
|
272
|
+
ref = str(issue_number)
|
|
132
273
|
if repo_full_name:
|
|
133
|
-
|
|
274
|
+
ref = "%s#%s" % (repo_full_name, issue_number)
|
|
275
|
+
args = ["start", ref, "--pr", "--detach"]
|
|
134
276
|
summary = "issue #%s opened in %s" % (issue_number, repo_full_name)
|
|
135
277
|
success = run_loki_command(args, dry_run=dry_run)
|
|
136
278
|
status = "fired" if success else "error"
|
|
@@ -140,7 +282,7 @@ def handle_issues_event(payload, dry_run=False):
|
|
|
140
282
|
|
|
141
283
|
|
|
142
284
|
def handle_pull_request_event(payload, dry_run=False):
|
|
143
|
-
"""Handle pull_request event: synchronize -> loki
|
|
285
|
+
"""Handle pull_request event: synchronize -> loki start <pr-ref> --detach."""
|
|
144
286
|
action = payload.get("action", "")
|
|
145
287
|
if action != "synchronize":
|
|
146
288
|
return None, "skipped (action=%s)" % action
|
|
@@ -148,11 +290,16 @@ def handle_pull_request_event(payload, dry_run=False):
|
|
|
148
290
|
pr_number = pr.get("number")
|
|
149
291
|
repo = payload.get("repository", {})
|
|
150
292
|
repo_full_name = repo.get("full_name", "")
|
|
151
|
-
if
|
|
293
|
+
if pr_number is None:
|
|
152
294
|
return None, "skipped (no PR number)"
|
|
153
|
-
|
|
295
|
+
if not valid_issue_number(pr_number):
|
|
296
|
+
return None, "rejected (invalid PR number)"
|
|
297
|
+
if repo_full_name and not valid_repo_full_name(repo_full_name):
|
|
298
|
+
return None, "rejected (invalid repository full_name)"
|
|
299
|
+
ref = str(pr_number)
|
|
154
300
|
if repo_full_name:
|
|
155
|
-
|
|
301
|
+
ref = "%s#%s" % (repo_full_name, pr_number)
|
|
302
|
+
args = ["start", ref, "--detach"]
|
|
156
303
|
summary = "PR #%s synchronized in %s" % (pr_number, repo_full_name)
|
|
157
304
|
success = run_loki_command(args, dry_run=dry_run)
|
|
158
305
|
status = "fired" if success else "error"
|
|
@@ -162,7 +309,7 @@ def handle_pull_request_event(payload, dry_run=False):
|
|
|
162
309
|
|
|
163
310
|
|
|
164
311
|
def handle_workflow_run_event(payload, dry_run=False):
|
|
165
|
-
"""Handle workflow_run event: completed+failure -> loki
|
|
312
|
+
"""Handle workflow_run event: completed+failure -> loki start with context."""
|
|
166
313
|
action = payload.get("action", "")
|
|
167
314
|
if action != "completed":
|
|
168
315
|
return None, "skipped (action=%s)" % action
|
|
@@ -174,8 +321,9 @@ def handle_workflow_run_event(payload, dry_run=False):
|
|
|
174
321
|
repo = payload.get("repository", {})
|
|
175
322
|
repo_full_name = repo.get("full_name", "")
|
|
176
323
|
summary = "workflow '%s' failed in %s" % (wf_name, repo_full_name)
|
|
177
|
-
#
|
|
178
|
-
|
|
324
|
+
# CI-failure context: re-run the current spec in the working directory so
|
|
325
|
+
# the agent can repair the failing build. No issue/PR ref to attach here.
|
|
326
|
+
args = ["start", "--detach"]
|
|
179
327
|
success = run_loki_command(args, dry_run=dry_run)
|
|
180
328
|
status = "fired" if success else "error"
|
|
181
329
|
if success:
|
|
@@ -183,11 +331,123 @@ def handle_workflow_run_event(payload, dry_run=False):
|
|
|
183
331
|
return summary, status
|
|
184
332
|
|
|
185
333
|
|
|
334
|
+
# Map GitHub event name -> handler. Keeps do_POST routing declarative.
|
|
335
|
+
EVENT_HANDLERS = {
|
|
336
|
+
"issues": handle_issues_event,
|
|
337
|
+
"pull_request": handle_pull_request_event,
|
|
338
|
+
"workflow_run": handle_workflow_run_event,
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def dispatch_event(event_type, payload, dry_run=False):
|
|
343
|
+
"""Route one webhook event to its handler and log the outcome.
|
|
344
|
+
|
|
345
|
+
Runs on a worker thread. Returns (summary, status). Any handler exception
|
|
346
|
+
is caught and logged so one bad payload cannot kill a worker.
|
|
347
|
+
"""
|
|
348
|
+
action = payload.get("action", "")
|
|
349
|
+
handler = EVENT_HANDLERS.get(event_type)
|
|
350
|
+
if handler is None:
|
|
351
|
+
status = "unsupported event: %s" % event_type
|
|
352
|
+
log_event(event_type, action, "", status)
|
|
353
|
+
return None, status
|
|
354
|
+
try:
|
|
355
|
+
summary, status = handler(payload, dry_run=dry_run)
|
|
356
|
+
except Exception as e: # defensive: never let a worker die on bad input
|
|
357
|
+
logging.exception("Handler for %s raised: %s", event_type, e)
|
|
358
|
+
summary, status = None, "error"
|
|
359
|
+
log_event(event_type, action, summary or "", status)
|
|
360
|
+
return summary, status
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
class Dispatcher:
|
|
364
|
+
"""Bounded worker pool that drains webhook events off a queue.
|
|
365
|
+
|
|
366
|
+
The HTTP handler enqueues work and returns immediately, so the listener
|
|
367
|
+
never blocks on a slow dispatch. A fixed number of worker threads drain the
|
|
368
|
+
queue; if the queue is full the handler is told to shed load (503) so a
|
|
369
|
+
webhook storm cannot fork unbounded builds.
|
|
370
|
+
"""
|
|
371
|
+
|
|
372
|
+
def __init__(self, workers=DEFAULT_WORKERS, queue_size=DEFAULT_QUEUE_SIZE,
|
|
373
|
+
dry_run=False, dedup_size=DEFAULT_DEDUP_SIZE):
|
|
374
|
+
self.dry_run = dry_run
|
|
375
|
+
self.queue = queue.Queue(maxsize=max(1, queue_size))
|
|
376
|
+
# Idempotency: remember recently seen GitHub delivery IDs so a
|
|
377
|
+
# redelivered webhook (GitHub retries on non-2xx, and operators can
|
|
378
|
+
# manually redeliver) does not dispatch the same build twice. Bounded
|
|
379
|
+
# FIFO so memory cannot grow without limit.
|
|
380
|
+
self._dedup_max = max(1, dedup_size)
|
|
381
|
+
self._seen_deliveries = collections.OrderedDict()
|
|
382
|
+
self._dedup_lock = threading.Lock()
|
|
383
|
+
self._threads = []
|
|
384
|
+
self._stop = threading.Event()
|
|
385
|
+
for i in range(max(1, workers)):
|
|
386
|
+
t = threading.Thread(
|
|
387
|
+
target=self._worker,
|
|
388
|
+
name="loki-trigger-worker-%d" % i,
|
|
389
|
+
daemon=True,
|
|
390
|
+
)
|
|
391
|
+
t.start()
|
|
392
|
+
self._threads.append(t)
|
|
393
|
+
|
|
394
|
+
def seen_delivery(self, delivery_id):
|
|
395
|
+
"""Return True if this delivery_id was already accepted (idempotency).
|
|
396
|
+
|
|
397
|
+
Records the id as seen as a side effect when it is new. A falsy
|
|
398
|
+
delivery_id (header absent) is never deduplicated, so requests without
|
|
399
|
+
a delivery id always fall through to normal handling.
|
|
400
|
+
"""
|
|
401
|
+
if not delivery_id:
|
|
402
|
+
return False
|
|
403
|
+
with self._dedup_lock:
|
|
404
|
+
if delivery_id in self._seen_deliveries:
|
|
405
|
+
# Do NOT refresh recency here. If a duplicate hit moved the id to
|
|
406
|
+
# the most-recent end, a flood of one valid (authenticated)
|
|
407
|
+
# duplicate id could keep it pinned and evict up to dedup_max
|
|
408
|
+
# genuinely-recent ids, letting real redeliveries slip through.
|
|
409
|
+
# Insertion order is the eviction policy; duplicates leave it
|
|
410
|
+
# unchanged.
|
|
411
|
+
return True
|
|
412
|
+
self._seen_deliveries[delivery_id] = True
|
|
413
|
+
while len(self._seen_deliveries) > self._dedup_max:
|
|
414
|
+
self._seen_deliveries.popitem(last=False)
|
|
415
|
+
return False
|
|
416
|
+
|
|
417
|
+
def submit(self, event_type, payload):
|
|
418
|
+
"""Enqueue an event. Returns True if accepted, False if the queue is full."""
|
|
419
|
+
try:
|
|
420
|
+
self.queue.put_nowait((event_type, payload))
|
|
421
|
+
return True
|
|
422
|
+
except queue.Full:
|
|
423
|
+
return False
|
|
424
|
+
|
|
425
|
+
def _worker(self):
|
|
426
|
+
while not self._stop.is_set():
|
|
427
|
+
try:
|
|
428
|
+
item = self.queue.get(timeout=0.5)
|
|
429
|
+
except queue.Empty:
|
|
430
|
+
continue
|
|
431
|
+
try:
|
|
432
|
+
event_type, payload = item
|
|
433
|
+
dispatch_event(event_type, payload, dry_run=self.dry_run)
|
|
434
|
+
finally:
|
|
435
|
+
self.queue.task_done()
|
|
436
|
+
|
|
437
|
+
def shutdown(self):
|
|
438
|
+
self._stop.set()
|
|
439
|
+
|
|
440
|
+
|
|
186
441
|
class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
187
442
|
"""HTTP request handler for GitHub webhooks."""
|
|
188
443
|
|
|
444
|
+
# Set on the class in main() before the server starts.
|
|
189
445
|
dry_run = False
|
|
190
446
|
secret = ""
|
|
447
|
+
dispatcher = None
|
|
448
|
+
|
|
449
|
+
# Cap the body we will read so a huge POST cannot exhaust memory.
|
|
450
|
+
MAX_BODY_BYTES = 5 * 1024 * 1024
|
|
191
451
|
|
|
192
452
|
def log_message(self, format, *args):
|
|
193
453
|
logging.info("%s - %s", self.address_string(), format % args)
|
|
@@ -202,6 +462,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
202
462
|
"dry_run": self.dry_run,
|
|
203
463
|
"port": config.get("port", 7373),
|
|
204
464
|
"enabled_events": config.get("enabled_events", []),
|
|
465
|
+
"secret_configured": bool(self.secret),
|
|
205
466
|
})
|
|
206
467
|
else:
|
|
207
468
|
self._send_json(404, {"error": "not found"})
|
|
@@ -211,14 +472,64 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
211
472
|
self._send_json(404, {"error": "not found"})
|
|
212
473
|
return
|
|
213
474
|
|
|
214
|
-
|
|
215
|
-
|
|
475
|
+
try:
|
|
476
|
+
content_length = int(self.headers.get("Content-Length", 0))
|
|
477
|
+
except (TypeError, ValueError):
|
|
478
|
+
self._send_json(400, {"error": "invalid Content-Length"})
|
|
479
|
+
return
|
|
480
|
+
if content_length < 0 or content_length > self.MAX_BODY_BYTES:
|
|
481
|
+
self._send_json(413, {"error": "payload too large"})
|
|
482
|
+
return
|
|
483
|
+
|
|
484
|
+
# L3 fix: bound the body read so a slow or under-delivering client (a
|
|
485
|
+
# slow-loris that drips or never finishes the body) cannot tie up a
|
|
486
|
+
# worker thread BEFORE authentication. A stalled read raises
|
|
487
|
+
# socket.timeout; we drop the connection rather than block forever.
|
|
488
|
+
prev_timeout = self.connection.gettimeout()
|
|
489
|
+
self.connection.settimeout(BODY_READ_TIMEOUT_SECONDS)
|
|
490
|
+
try:
|
|
491
|
+
body = self.rfile.read(content_length)
|
|
492
|
+
except (socket.timeout, TimeoutError, ConnectionError, OSError):
|
|
493
|
+
logging.warning(
|
|
494
|
+
"Dropping slow/incomplete webhook body from %s",
|
|
495
|
+
self.address_string(),
|
|
496
|
+
)
|
|
497
|
+
try:
|
|
498
|
+
self._send_json(408, {"error": "request body timeout"})
|
|
499
|
+
except (BrokenPipeError, ConnectionResetError, OSError):
|
|
500
|
+
pass
|
|
501
|
+
return
|
|
502
|
+
finally:
|
|
503
|
+
try:
|
|
504
|
+
self.connection.settimeout(prev_timeout)
|
|
505
|
+
except OSError:
|
|
506
|
+
pass
|
|
507
|
+
if len(body) != content_length:
|
|
508
|
+
# Client closed before delivering the declared body. Refuse rather
|
|
509
|
+
# than authenticate a truncated payload.
|
|
510
|
+
self._send_json(400, {"error": "incomplete request body"})
|
|
511
|
+
return
|
|
216
512
|
|
|
217
513
|
event_type = self.headers.get("X-GitHub-Event", "")
|
|
218
514
|
signature = self.headers.get("X-Hub-Signature-256", "")
|
|
515
|
+
delivery_id = self.headers.get("X-GitHub-Delivery", "")
|
|
516
|
+
|
|
517
|
+
# Defect 1 fix: refuse to dispatch when no secret is configured. The
|
|
518
|
+
# server stays up for ops endpoints, but webhooks are rejected with an
|
|
519
|
+
# audit line. We never silently accept-all.
|
|
520
|
+
if not self.secret:
|
|
521
|
+
logging.warning(
|
|
522
|
+
"Rejecting webhook from %s: no secret configured "
|
|
523
|
+
"(set --secret or config.secret to enable dispatch)",
|
|
524
|
+
self.address_string(),
|
|
525
|
+
)
|
|
526
|
+
log_event(event_type, "", "", "rejected (no secret configured)")
|
|
527
|
+
self._send_json(503, {"error": "webhook secret not configured"})
|
|
528
|
+
return
|
|
219
529
|
|
|
220
530
|
if not validate_signature(self.secret, body, signature):
|
|
221
|
-
logging.warning("Invalid webhook signature")
|
|
531
|
+
logging.warning("Invalid webhook signature from %s", self.address_string())
|
|
532
|
+
log_event(event_type, "", "", "rejected (invalid signature)")
|
|
222
533
|
self._send_json(401, {"error": "invalid signature"})
|
|
223
534
|
return
|
|
224
535
|
|
|
@@ -227,22 +538,52 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
227
538
|
except json.JSONDecodeError:
|
|
228
539
|
self._send_json(400, {"error": "invalid JSON"})
|
|
229
540
|
return
|
|
541
|
+
if not isinstance(payload, dict):
|
|
542
|
+
self._send_json(400, {"error": "payload must be a JSON object"})
|
|
543
|
+
return
|
|
230
544
|
|
|
231
545
|
action = payload.get("action", "")
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
if event_type == "issues":
|
|
236
|
-
summary, status = handle_issues_event(payload, dry_run=self.dry_run)
|
|
237
|
-
elif event_type == "pull_request":
|
|
238
|
-
summary, status = handle_pull_request_event(payload, dry_run=self.dry_run)
|
|
239
|
-
elif event_type == "workflow_run":
|
|
240
|
-
summary, status = handle_workflow_run_event(payload, dry_run=self.dry_run)
|
|
241
|
-
else:
|
|
546
|
+
|
|
547
|
+
if event_type not in EVENT_HANDLERS:
|
|
242
548
|
status = "unsupported event: %s" % event_type
|
|
549
|
+
log_event(event_type, action, "", status)
|
|
550
|
+
self._send_json(
|
|
551
|
+
200, {"event": event_type, "action": action, "status": status}
|
|
552
|
+
)
|
|
553
|
+
return
|
|
243
554
|
|
|
244
|
-
|
|
245
|
-
|
|
555
|
+
# Idempotency: a redelivered webhook (same X-GitHub-Delivery) must not
|
|
556
|
+
# dispatch the same build twice. Checked only after authentication so an
|
|
557
|
+
# attacker cannot poison the cache. Returns 200 so GitHub stops retrying.
|
|
558
|
+
if self.dispatcher.seen_delivery(delivery_id):
|
|
559
|
+
logging.info(
|
|
560
|
+
"Duplicate delivery %s (%s); skipping re-dispatch",
|
|
561
|
+
delivery_id,
|
|
562
|
+
event_type,
|
|
563
|
+
)
|
|
564
|
+
log_event(event_type, action, "", "duplicate (delivery %s)" % delivery_id)
|
|
565
|
+
self._send_json(
|
|
566
|
+
200,
|
|
567
|
+
{"event": event_type, "action": action, "status": "duplicate"},
|
|
568
|
+
)
|
|
569
|
+
return
|
|
570
|
+
|
|
571
|
+
# Hand off to the bounded worker queue so the listener never blocks.
|
|
572
|
+
accepted = self.dispatcher.submit(event_type, payload)
|
|
573
|
+
if not accepted:
|
|
574
|
+
logging.warning(
|
|
575
|
+
"Queue full; shedding webhook %s from %s",
|
|
576
|
+
event_type,
|
|
577
|
+
self.address_string(),
|
|
578
|
+
)
|
|
579
|
+
log_event(event_type, action, "", "rejected (queue full)")
|
|
580
|
+
self._send_json(503, {"error": "server busy, retry later"})
|
|
581
|
+
return
|
|
582
|
+
|
|
583
|
+
# 202 Accepted: queued for processing, not yet fired.
|
|
584
|
+
self._send_json(
|
|
585
|
+
202, {"event": event_type, "action": action, "status": "queued"}
|
|
586
|
+
)
|
|
246
587
|
|
|
247
588
|
def _send_json(self, code, data):
|
|
248
589
|
body = json.dumps(data).encode("utf-8")
|
|
@@ -250,7 +591,18 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
|
|
250
591
|
self.send_header("Content-Type", "application/json")
|
|
251
592
|
self.send_header("Content-Length", str(len(body)))
|
|
252
593
|
self.end_headers()
|
|
253
|
-
|
|
594
|
+
try:
|
|
595
|
+
self.wfile.write(body)
|
|
596
|
+
except (BrokenPipeError, ConnectionResetError):
|
|
597
|
+
pass
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
class ThreadingWebhookServer(socketserver.ThreadingMixIn,
|
|
601
|
+
http.server.HTTPServer):
|
|
602
|
+
"""Threaded HTTP server so a slow request never serializes the listener."""
|
|
603
|
+
|
|
604
|
+
daemon_threads = True
|
|
605
|
+
allow_reuse_address = True
|
|
254
606
|
|
|
255
607
|
|
|
256
608
|
def write_pid_file():
|
|
@@ -267,6 +619,8 @@ def main():
|
|
|
267
619
|
parser.add_argument("--port", type=int, default=None, help="Port to listen on (default: 7373)")
|
|
268
620
|
parser.add_argument("--secret", default=None, help="GitHub webhook secret for HMAC validation")
|
|
269
621
|
parser.add_argument("--dry-run", action="store_true", help="Preview triggers without running loki")
|
|
622
|
+
parser.add_argument("--workers", type=int, default=None, help="Worker threads draining the dispatch queue")
|
|
623
|
+
parser.add_argument("--queue-size", type=int, default=None, help="Max in-flight queued dispatches")
|
|
270
624
|
args = parser.parse_args()
|
|
271
625
|
|
|
272
626
|
logging.basicConfig(
|
|
@@ -279,28 +633,50 @@ def main():
|
|
|
279
633
|
port = args.port if args.port is not None else config.get("port", 7373)
|
|
280
634
|
secret = args.secret if args.secret is not None else config.get("secret", "")
|
|
281
635
|
dry_run = args.dry_run or config.get("dry_run", False)
|
|
636
|
+
workers = args.workers if args.workers is not None else config.get("workers", DEFAULT_WORKERS)
|
|
637
|
+
queue_size = args.queue_size if args.queue_size is not None else config.get("queue_size", DEFAULT_QUEUE_SIZE)
|
|
282
638
|
|
|
283
|
-
#
|
|
639
|
+
# Allow GITHUB_WEBHOOK_SECRET as a non-CLI source so the secret need not
|
|
640
|
+
# land in argv or the config file.
|
|
641
|
+
if not secret:
|
|
642
|
+
secret = os.environ.get("GITHUB_WEBHOOK_SECRET", "")
|
|
643
|
+
|
|
644
|
+
# Persist resolved values, but never write the secret to disk.
|
|
284
645
|
config["port"] = port
|
|
285
|
-
config["secret"] = secret
|
|
286
646
|
config["dry_run"] = dry_run
|
|
647
|
+
config["workers"] = workers
|
|
648
|
+
config["queue_size"] = queue_size
|
|
649
|
+
config["secret"] = ""
|
|
287
650
|
save_config(config)
|
|
288
651
|
|
|
652
|
+
dispatcher = Dispatcher(workers=workers, queue_size=queue_size, dry_run=dry_run)
|
|
653
|
+
|
|
289
654
|
WebhookHandler.dry_run = dry_run
|
|
290
655
|
WebhookHandler.secret = secret
|
|
656
|
+
WebhookHandler.dispatcher = dispatcher
|
|
291
657
|
|
|
292
|
-
server =
|
|
658
|
+
server = ThreadingWebhookServer(("", port), WebhookHandler)
|
|
293
659
|
write_pid_file()
|
|
294
660
|
|
|
295
661
|
mode_label = " [DRY-RUN]" if dry_run else ""
|
|
296
662
|
logging.info("Loki trigger server starting on port %d%s", port, mode_label)
|
|
297
663
|
logging.info("Webhook endpoint: POST http://localhost:%d/webhook", port)
|
|
298
664
|
logging.info("Health check: GET http://localhost:%d/health", port)
|
|
665
|
+
logging.info("Workers: %d, queue size: %d", workers, queue_size)
|
|
666
|
+
if not secret:
|
|
667
|
+
logging.warning(
|
|
668
|
+
"No webhook secret configured: ALL webhooks will be rejected with "
|
|
669
|
+
"503. Set --secret, config.secret, or GITHUB_WEBHOOK_SECRET to "
|
|
670
|
+
"enable dispatch."
|
|
671
|
+
)
|
|
299
672
|
|
|
300
673
|
try:
|
|
301
674
|
server.serve_forever()
|
|
302
675
|
except KeyboardInterrupt:
|
|
303
676
|
logging.info("Trigger server stopped.")
|
|
677
|
+
finally:
|
|
678
|
+
dispatcher.shutdown()
|
|
679
|
+
server.server_close()
|
|
304
680
|
pid_path = get_loki_dir() / "server.pid"
|
|
305
681
|
pid_path.unlink(missing_ok=True)
|
|
306
682
|
|