@caddis/cli 0.4.13 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundles/antigravity-plugin/plugin.json +1 -1
- package/bundles/antigravity-plugin/repo_freshness.py +146 -0
- package/bundles/antigravity-plugin/scripts/caddis-run.ps1 +11 -0
- package/bundles/antigravity-plugin/scripts/caddis-run.sh +9 -0
- package/bundles/antigravity-plugin/scripts/caddis_gate.py +24 -3
- package/bundles/antigravity-plugin/scripts/caddis_governor.py +542 -0
- package/bundles/antigravity-plugin/scripts/caddis_host_guard.py +689 -0
- package/bundles/antigravity-plugin/scripts/caddis_lanes.py +1632 -0
- package/bundles/antigravity-plugin/scripts/install-host-guard.ps1 +393 -0
- package/bundles/antigravity-plugin/skills/feature-plan/SKILL.md +30 -4
- package/bundles/antigravity-plugin/skills/implement/SKILL.md +48 -1
- package/bundles/antigravity-plugin/warm_start_agy.py +38 -4
- package/bundles/manifest.json +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,689 @@
|
|
|
1
|
+
"""caddis_host_guard — end runaway agent-started processes after sustained host saturation.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python caddis_host_guard.py run [--budget PATH] [--max-ticks N]
|
|
5
|
+
python caddis_host_guard.py once [--budget PATH]
|
|
6
|
+
python caddis_host_guard.py summary [--hours N]
|
|
7
|
+
|
|
8
|
+
Layer 3 of caddis resource protection (layer 1: BelowNormal priority; layer 2: Job Objects through
|
|
9
|
+
caddis_governor.py). It covers what layer 2 cannot: agents started without `caddis-run`, and a cap
|
|
10
|
+
set too loose. It runs as a LocalSystem service, because some agent sandboxes run commands under a
|
|
11
|
+
separate local account.
|
|
12
|
+
|
|
13
|
+
Every `sample_seconds` it samples host CPU and free memory. Every 2 seconds it records which
|
|
14
|
+
processes descend from a root agent image (claude.exe, codex.exe, agy.exe). A process stays marked
|
|
15
|
+
after its parent exits; creation times guard against a reused process id. After `sustain_seconds`
|
|
16
|
+
above `host_cpu_percent`, or below `min_free_memory_mb`, it ends the heaviest marked processes until
|
|
17
|
+
the excess is covered.
|
|
18
|
+
|
|
19
|
+
It never ends a root agent image, and never ends a process it has not marked (CI runners, the
|
|
20
|
+
desktop, services, dev servers); it reports those instead. Its settings come only from the
|
|
21
|
+
admin-owned machine budget, never from a user-writable file.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import argparse
|
|
26
|
+
import importlib.util
|
|
27
|
+
import json
|
|
28
|
+
import os
|
|
29
|
+
import sys
|
|
30
|
+
import time
|
|
31
|
+
from dataclasses import dataclass, field
|
|
32
|
+
from datetime import datetime, timezone
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
EXIT_CONFIG = 3
|
|
36
|
+
LINEAGE_POLL_SECONDS = 2.0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class GuardError(Exception):
|
|
40
|
+
"""The guard's configuration cannot be used."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Proc:
|
|
45
|
+
pid: int
|
|
46
|
+
ppid: int
|
|
47
|
+
exe: str # lowercased basename
|
|
48
|
+
created: int # creation time in 100 ns units; 0 when unknown
|
|
49
|
+
governor: bool = False
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class GuardConfig:
|
|
54
|
+
mode: str
|
|
55
|
+
host_cpu_percent: int
|
|
56
|
+
sustain_seconds: int
|
|
57
|
+
sample_seconds: int
|
|
58
|
+
min_free_memory_mb: int
|
|
59
|
+
root_images: tuple[str, ...]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
DEFAULT_GUARD = GuardConfig("enforce", 85, 60, 10, 4096, ("claude.exe", "codex.exe", "agy.exe"))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class Sample:
|
|
67
|
+
ts: float
|
|
68
|
+
host_cpu: float
|
|
69
|
+
free_mb: int
|
|
70
|
+
cpu_s: dict[int, float]
|
|
71
|
+
mem_mb: dict[int, int] = field(default_factory=dict)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class Action:
|
|
76
|
+
kind: str # "end", "would-end" or "report"
|
|
77
|
+
pid: int | None
|
|
78
|
+
exe: str | None
|
|
79
|
+
reason: str
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# ── configuration ───────────────────────────────────────────────────────────
|
|
83
|
+
|
|
84
|
+
def _governor():
|
|
85
|
+
"""caddis_governor.py sits beside this file in the source tree, the plugin bundle and ProgramData."""
|
|
86
|
+
path = Path(__file__).resolve().with_name("caddis_governor.py")
|
|
87
|
+
if not path.is_file():
|
|
88
|
+
raise GuardError(f"caddis_governor.py not found beside {Path(__file__).name}")
|
|
89
|
+
spec = importlib.util.spec_from_file_location("_caddis_governor_for_guard", path)
|
|
90
|
+
mod = importlib.util.module_from_spec(spec)
|
|
91
|
+
sys.modules[spec.name] = mod # registered BEFORE exec: @dataclass looks the module up
|
|
92
|
+
spec.loader.exec_module(mod)
|
|
93
|
+
return mod
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _int(key: str, value: object, low: int, high: int) -> int:
|
|
97
|
+
if isinstance(value, bool) or not isinstance(value, int) or not low <= value <= high:
|
|
98
|
+
raise GuardError(f"invalid [guard] {key}: {value!r} (expected an integer {low}..{high})")
|
|
99
|
+
return value
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def load_guard_config(path: Path | None = None) -> GuardConfig:
|
|
103
|
+
"""The [guard] table of the MACHINE budget. A missing file means defaults; a broken one is an error."""
|
|
104
|
+
file = path or _governor().machine_budget_path()
|
|
105
|
+
if not file.is_file():
|
|
106
|
+
return DEFAULT_GUARD
|
|
107
|
+
try:
|
|
108
|
+
import tomllib
|
|
109
|
+
except ImportError as exc:
|
|
110
|
+
raise GuardError(f"{file} exists but this Python has no tomllib (need 3.11+)") from exc
|
|
111
|
+
try:
|
|
112
|
+
table = tomllib.loads(file.read_text(encoding="utf-8")).get("guard", {})
|
|
113
|
+
except Exception as exc:
|
|
114
|
+
raise GuardError(f"cannot parse {file}: {exc}") from exc
|
|
115
|
+
if not isinstance(table, dict):
|
|
116
|
+
raise GuardError(f"[guard] in {file} is not a table")
|
|
117
|
+
d = DEFAULT_GUARD
|
|
118
|
+
mode = table.get("mode", d.mode)
|
|
119
|
+
if mode not in ("enforce", "report"):
|
|
120
|
+
raise GuardError(f"invalid [guard] mode: {mode!r} (expected 'enforce' or 'report')")
|
|
121
|
+
roots = table.get("root_images", list(d.root_images))
|
|
122
|
+
if not isinstance(roots, list) or not roots or not all(isinstance(r, str) and r.strip() for r in roots):
|
|
123
|
+
raise GuardError(f"invalid [guard] root_images: {roots!r}")
|
|
124
|
+
return GuardConfig(
|
|
125
|
+
mode=mode,
|
|
126
|
+
host_cpu_percent=_int("host_cpu_percent", table.get("host_cpu_percent", d.host_cpu_percent), 50, 99),
|
|
127
|
+
sustain_seconds=_int("sustain_seconds", table.get("sustain_seconds", d.sustain_seconds), 1, 3600),
|
|
128
|
+
sample_seconds=_int("sample_seconds", table.get("sample_seconds", d.sample_seconds), 1, 600),
|
|
129
|
+
min_free_memory_mb=_int("min_free_memory_mb", table.get("min_free_memory_mb", d.min_free_memory_mb),
|
|
130
|
+
0, 1_048_576),
|
|
131
|
+
root_images=tuple(r.strip().lower() for r in roots),
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# ── lineage and decisions (pure) ────────────────────────────────────────────
|
|
136
|
+
|
|
137
|
+
class Lineage:
|
|
138
|
+
"""Which live processes descend from a root agent image. A mark persists after the parent exits."""
|
|
139
|
+
|
|
140
|
+
def __init__(self) -> None:
|
|
141
|
+
self.marked: dict[int, int] = {} # pid -> creation time when marked
|
|
142
|
+
self.verified: set[int] = set()
|
|
143
|
+
self.root_images: tuple[str, ...] = DEFAULT_GUARD.root_images
|
|
144
|
+
|
|
145
|
+
def update(self, procs: dict[int, Proc], roots: tuple[str, ...]) -> None:
|
|
146
|
+
self.root_images = roots
|
|
147
|
+
self.verified = set()
|
|
148
|
+
for pid in list(self.marked):
|
|
149
|
+
p = procs.get(pid)
|
|
150
|
+
if p is None or (p.created != 0 and p.created != self.marked[pid]):
|
|
151
|
+
del self.marked[pid] # exited, or the id now belongs to a different process
|
|
152
|
+
elif p.created == self.marked[pid]:
|
|
153
|
+
self.verified.add(pid)
|
|
154
|
+
changed = True
|
|
155
|
+
while changed:
|
|
156
|
+
changed = False
|
|
157
|
+
for p in procs.values():
|
|
158
|
+
if p.pid in self.marked or p.created == 0:
|
|
159
|
+
continue
|
|
160
|
+
if p.exe in roots or (p.ppid in self.verified and self.marked[p.ppid] <= p.created):
|
|
161
|
+
self.marked[p.pid] = p.created
|
|
162
|
+
self.verified.add(p.pid)
|
|
163
|
+
changed = True
|
|
164
|
+
|
|
165
|
+
def is_agent(self, pid: int) -> bool:
|
|
166
|
+
return pid in self.marked
|
|
167
|
+
|
|
168
|
+
@staticmethod
|
|
169
|
+
def is_root(proc: Proc, roots: tuple[str, ...]) -> bool:
|
|
170
|
+
return proc.exe in roots
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def protected_ancestors(procs: dict[int, Proc], roots: tuple[str, ...]) -> set[int]:
|
|
174
|
+
"""Protect roots, job owners, and every ancestor that could end them indirectly."""
|
|
175
|
+
protected: set[int] = set()
|
|
176
|
+
for proc in procs.values():
|
|
177
|
+
if not (proc.exe in roots or proc.governor or proc.exe == "caddis_governor.exe"):
|
|
178
|
+
continue
|
|
179
|
+
current = proc
|
|
180
|
+
while current.pid not in protected:
|
|
181
|
+
protected.add(current.pid)
|
|
182
|
+
parent = procs.get(current.ppid)
|
|
183
|
+
if parent is None or (parent.created and current.created and parent.created > current.created):
|
|
184
|
+
break
|
|
185
|
+
current = parent
|
|
186
|
+
return protected
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def decide(window: list[Sample], lineage: Lineage, procs: dict[int, Proc], cfg: GuardConfig,
|
|
190
|
+
cpu_count: int) -> list[Action]:
|
|
191
|
+
"""What to do about `window` (samples, oldest first). Pure: it never touches a process."""
|
|
192
|
+
if not window:
|
|
193
|
+
return []
|
|
194
|
+
cutoff = window[-1].ts - cfg.sustain_seconds
|
|
195
|
+
before = next((sample for sample in reversed(window) if sample.ts < cutoff), None)
|
|
196
|
+
window = [sample for sample in window if sample.ts >= cutoff]
|
|
197
|
+
if before is not None and window and window[0].ts > cutoff:
|
|
198
|
+
after = window[0]
|
|
199
|
+
fraction = (cutoff - before.ts) / (after.ts - before.ts)
|
|
200
|
+
cpu_s = {pid: before.cpu_s.get(pid, value) + (value - before.cpu_s.get(pid, value)) * fraction
|
|
201
|
+
for pid, value in after.cpu_s.items()}
|
|
202
|
+
window.insert(0, Sample(cutoff, after.host_cpu, after.free_mb, cpu_s, after.mem_mb))
|
|
203
|
+
first, last = window[0], window[-1]
|
|
204
|
+
span = last.ts - first.ts
|
|
205
|
+
saturated = (len(window) >= 2 and span >= cfg.sustain_seconds
|
|
206
|
+
and all(s.host_cpu > cfg.host_cpu_percent for s in window))
|
|
207
|
+
low_memory = last.free_mb < cfg.min_free_memory_mb
|
|
208
|
+
if not (saturated or low_memory):
|
|
209
|
+
return []
|
|
210
|
+
|
|
211
|
+
protected = protected_ancestors(procs, cfg.root_images)
|
|
212
|
+
candidates = [pid for pid, p in procs.items()
|
|
213
|
+
if pid in lineage.verified and lineage.marked.get(pid) == p.created
|
|
214
|
+
and pid not in protected]
|
|
215
|
+
mean_cpu = sum(s.host_cpu for s in window) / len(window)
|
|
216
|
+
kind = "end" if cfg.mode == "enforce" else "would-end"
|
|
217
|
+
chosen: list[Action] = []
|
|
218
|
+
|
|
219
|
+
if low_memory and last.mem_mb:
|
|
220
|
+
by_mem = sorted((pid for pid in candidates if last.mem_mb.get(pid, 0) > 0),
|
|
221
|
+
key=lambda pid: last.mem_mb[pid], reverse=True)
|
|
222
|
+
if by_mem:
|
|
223
|
+
pid = by_mem[0]
|
|
224
|
+
chosen.append(Action(kind, pid, procs[pid].exe,
|
|
225
|
+
f"free memory {last.free_mb} MB; process uses {last.mem_mb[pid]} MB"))
|
|
226
|
+
return chosen
|
|
227
|
+
|
|
228
|
+
if span <= 0:
|
|
229
|
+
return [Action("report", None, None,
|
|
230
|
+
"host saturated but no agent-descended process is using CPU")]
|
|
231
|
+
|
|
232
|
+
used = {pid: max(0.0, last.cpu_s.get(pid, 0.0) - first.cpu_s.get(pid, 0.0)) for pid in candidates}
|
|
233
|
+
ranked = sorted((pid for pid in candidates if used[pid] > 0), key=lambda pid: used[pid], reverse=True)
|
|
234
|
+
excess = (mean_cpu - cfg.host_cpu_percent) / 100 * cpu_count
|
|
235
|
+
if low_memory:
|
|
236
|
+
excess = max(excess, 1.0)
|
|
237
|
+
covered = 0.0
|
|
238
|
+
for pid in ranked:
|
|
239
|
+
if covered >= excess:
|
|
240
|
+
break
|
|
241
|
+
cores = used[pid] / span
|
|
242
|
+
covered += cores
|
|
243
|
+
chosen.append(Action(kind, pid, procs[pid].exe,
|
|
244
|
+
f"host cpu {mean_cpu:.0f}% for {span:.0f}s; process used {cores:.2f} cores"))
|
|
245
|
+
if not chosen:
|
|
246
|
+
chosen.append(Action("report", None, None,
|
|
247
|
+
"host saturated but no agent-descended process is using CPU"))
|
|
248
|
+
return chosen
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# ── log ─────────────────────────────────────────────────────────────────────
|
|
252
|
+
|
|
253
|
+
def guard_log_path() -> Path:
|
|
254
|
+
override = os.environ.get("CADDIS_GUARD_LOG")
|
|
255
|
+
if override:
|
|
256
|
+
return Path(override)
|
|
257
|
+
if sys.platform == "win32":
|
|
258
|
+
return Path(os.environ.get("PROGRAMDATA", r"C:\ProgramData")) / "caddis" / "host-guard.jsonl"
|
|
259
|
+
return Path.home() / ".caddis" / "host-guard.jsonl"
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def append_log(path: Path, record: dict) -> None:
|
|
263
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
264
|
+
with path.open("a", encoding="utf-8") as fh:
|
|
265
|
+
fh.write(json.dumps(record, sort_keys=True) + "\n")
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def summarise(log: Path, hours: int = 24, now: float | None = None) -> str:
|
|
269
|
+
"""One line for SessionStart about recent guard actions, or "" when there were none. Never raises."""
|
|
270
|
+
try:
|
|
271
|
+
cutoff = (time.time() if now is None else now) - hours * 3600
|
|
272
|
+
hits = []
|
|
273
|
+
for line in log.read_text(encoding="utf-8").splitlines():
|
|
274
|
+
try:
|
|
275
|
+
rec = json.loads(line)
|
|
276
|
+
except ValueError:
|
|
277
|
+
continue
|
|
278
|
+
if rec.get("kind") in ("end", "end-failed", "would-end") and float(rec.get("ts", 0)) >= cutoff:
|
|
279
|
+
hits.append(rec)
|
|
280
|
+
if not hits:
|
|
281
|
+
return ""
|
|
282
|
+
ended = sum(len(r.get("killed", [])) for r in hits if r["kind"] == "end")
|
|
283
|
+
flagged = sum(r["kind"] == "would-end" for r in hits)
|
|
284
|
+
failed = sum(r["kind"] == "end-failed" or (r["kind"] == "end" and not r.get("killed"))
|
|
285
|
+
for r in hits)
|
|
286
|
+
counts = []
|
|
287
|
+
if ended:
|
|
288
|
+
counts.append(f"{ended} agent process(es) ended")
|
|
289
|
+
if flagged:
|
|
290
|
+
counts.append(f"{flagged} agent process(es) flagged")
|
|
291
|
+
if failed:
|
|
292
|
+
counts.append(f"{failed} end attempt(s) without confirmed termination")
|
|
293
|
+
last = hits[-1]
|
|
294
|
+
when = datetime.fromtimestamp(float(last["ts"]), timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
|
295
|
+
return (f"[caddis] host guard: {', '.join(counts)} in the last {hours}h "
|
|
296
|
+
f"(last: {last.get('exe')} pid {last.get('pid')} at {when}) — see {log}")
|
|
297
|
+
except Exception:
|
|
298
|
+
return ""
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
# ── Windows sampling (part 2) ───────────────────────────────────────────────
|
|
302
|
+
|
|
303
|
+
TH32CS_SNAPPROCESS = 0x2
|
|
304
|
+
PROCESS_TERMINATE = 0x1
|
|
305
|
+
PROCESS_VM_READ = 0x10
|
|
306
|
+
PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
|
|
307
|
+
ERROR_NO_MORE_FILES = 18
|
|
308
|
+
|
|
309
|
+
if sys.platform == "win32":
|
|
310
|
+
import ctypes
|
|
311
|
+
from ctypes import wintypes
|
|
312
|
+
|
|
313
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
314
|
+
psapi = ctypes.WinDLL("psapi", use_last_error=True)
|
|
315
|
+
ntdll = ctypes.WinDLL("ntdll")
|
|
316
|
+
|
|
317
|
+
class UNICODE_STRING(ctypes.Structure):
|
|
318
|
+
_fields_ = [("Length", wintypes.USHORT), ("MaximumLength", wintypes.USHORT),
|
|
319
|
+
("Buffer", ctypes.c_void_p)]
|
|
320
|
+
|
|
321
|
+
class FILETIME(ctypes.Structure):
|
|
322
|
+
_fields_ = [("dwLowDateTime", wintypes.DWORD), ("dwHighDateTime", wintypes.DWORD)]
|
|
323
|
+
|
|
324
|
+
class MEMORYSTATUSEX(ctypes.Structure):
|
|
325
|
+
_fields_ = [("dwLength", wintypes.DWORD), ("dwMemoryLoad", wintypes.DWORD),
|
|
326
|
+
("ullTotalPhys", ctypes.c_uint64), ("ullAvailPhys", ctypes.c_uint64),
|
|
327
|
+
("ullTotalPageFile", ctypes.c_uint64), ("ullAvailPageFile", ctypes.c_uint64),
|
|
328
|
+
("ullTotalVirtual", ctypes.c_uint64), ("ullAvailVirtual", ctypes.c_uint64),
|
|
329
|
+
("ullAvailExtendedVirtual", ctypes.c_uint64)]
|
|
330
|
+
|
|
331
|
+
class PROCESSENTRY32W(ctypes.Structure):
|
|
332
|
+
_fields_ = [("dwSize", wintypes.DWORD), ("cntUsage", wintypes.DWORD),
|
|
333
|
+
("th32ProcessID", wintypes.DWORD), ("th32DefaultHeapID", ctypes.c_size_t),
|
|
334
|
+
("th32ModuleID", wintypes.DWORD), ("cntThreads", wintypes.DWORD),
|
|
335
|
+
("th32ParentProcessID", wintypes.DWORD), ("pcPriClassBase", wintypes.LONG),
|
|
336
|
+
("dwFlags", wintypes.DWORD), ("szExeFile", wintypes.WCHAR * 260)]
|
|
337
|
+
|
|
338
|
+
class PROCESS_MEMORY_COUNTERS(ctypes.Structure):
|
|
339
|
+
_fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD),
|
|
340
|
+
("PeakWorkingSetSize", ctypes.c_size_t), ("WorkingSetSize", ctypes.c_size_t),
|
|
341
|
+
("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
|
|
342
|
+
("QuotaPagedPoolUsage", ctypes.c_size_t),
|
|
343
|
+
("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
|
|
344
|
+
("QuotaNonPagedPoolUsage", ctypes.c_size_t),
|
|
345
|
+
("PagefileUsage", ctypes.c_size_t), ("PeakPagefileUsage", ctypes.c_size_t)]
|
|
346
|
+
|
|
347
|
+
INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
|
|
348
|
+
|
|
349
|
+
kernel32.GetSystemTimes.argtypes = [ctypes.POINTER(FILETIME), ctypes.POINTER(FILETIME),
|
|
350
|
+
ctypes.POINTER(FILETIME)]
|
|
351
|
+
kernel32.GetSystemTimes.restype = wintypes.BOOL
|
|
352
|
+
kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]
|
|
353
|
+
kernel32.GlobalMemoryStatusEx.restype = wintypes.BOOL
|
|
354
|
+
kernel32.CreateToolhelp32Snapshot.argtypes = [wintypes.DWORD, wintypes.DWORD]
|
|
355
|
+
kernel32.CreateToolhelp32Snapshot.restype = wintypes.HANDLE
|
|
356
|
+
kernel32.Process32FirstW.argtypes = [wintypes.HANDLE, ctypes.POINTER(PROCESSENTRY32W)]
|
|
357
|
+
kernel32.Process32FirstW.restype = wintypes.BOOL
|
|
358
|
+
kernel32.Process32NextW.argtypes = [wintypes.HANDLE, ctypes.POINTER(PROCESSENTRY32W)]
|
|
359
|
+
kernel32.Process32NextW.restype = wintypes.BOOL
|
|
360
|
+
kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
|
361
|
+
kernel32.OpenProcess.restype = wintypes.HANDLE
|
|
362
|
+
kernel32.GetProcessTimes.argtypes = [wintypes.HANDLE, ctypes.POINTER(FILETIME), ctypes.POINTER(FILETIME),
|
|
363
|
+
ctypes.POINTER(FILETIME), ctypes.POINTER(FILETIME)]
|
|
364
|
+
kernel32.GetProcessTimes.restype = wintypes.BOOL
|
|
365
|
+
kernel32.QueryFullProcessImageNameW.argtypes = [wintypes.HANDLE, wintypes.DWORD,
|
|
366
|
+
wintypes.LPWSTR, ctypes.POINTER(wintypes.DWORD)]
|
|
367
|
+
kernel32.QueryFullProcessImageNameW.restype = wintypes.BOOL
|
|
368
|
+
ntdll.NtQueryInformationProcess.argtypes = [wintypes.HANDLE, wintypes.ULONG, ctypes.c_void_p,
|
|
369
|
+
wintypes.ULONG, ctypes.POINTER(wintypes.ULONG)]
|
|
370
|
+
ntdll.NtQueryInformationProcess.restype = wintypes.LONG
|
|
371
|
+
kernel32.TerminateProcess.argtypes = [wintypes.HANDLE, wintypes.UINT]
|
|
372
|
+
kernel32.TerminateProcess.restype = wintypes.BOOL
|
|
373
|
+
kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
|
|
374
|
+
kernel32.CloseHandle.restype = wintypes.BOOL
|
|
375
|
+
psapi.GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(PROCESS_MEMORY_COUNTERS),
|
|
376
|
+
wintypes.DWORD]
|
|
377
|
+
psapi.GetProcessMemoryInfo.restype = wintypes.BOOL
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _win_error(call: str) -> GuardError:
|
|
381
|
+
return GuardError(f"{call} failed: winerror {ctypes.get_last_error()}")
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _filetime_value(value: FILETIME) -> int:
|
|
385
|
+
return (int(value.dwHighDateTime) << 32) | int(value.dwLowDateTime)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _times_for_handle(handle: int) -> tuple[int, float] | None:
|
|
389
|
+
created = FILETIME()
|
|
390
|
+
exited = FILETIME()
|
|
391
|
+
kernel = FILETIME()
|
|
392
|
+
user = FILETIME()
|
|
393
|
+
if not kernel32.GetProcessTimes(handle, ctypes.byref(created), ctypes.byref(exited),
|
|
394
|
+
ctypes.byref(kernel), ctypes.byref(user)):
|
|
395
|
+
return None
|
|
396
|
+
cpu_s = (_filetime_value(kernel) + _filetime_value(user)) / 1e7
|
|
397
|
+
return _filetime_value(created), cpu_s
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def host_cpu_percent(prev: tuple[int, int, int] | None) -> tuple[float, tuple[int, int, int]]:
|
|
401
|
+
"""Return CPU use since `prev`, plus the current GetSystemTimes counters."""
|
|
402
|
+
idle = FILETIME()
|
|
403
|
+
kernel = FILETIME()
|
|
404
|
+
user = FILETIME()
|
|
405
|
+
if not kernel32.GetSystemTimes(ctypes.byref(idle), ctypes.byref(kernel), ctypes.byref(user)):
|
|
406
|
+
raise _win_error("GetSystemTimes")
|
|
407
|
+
current = (_filetime_value(idle), _filetime_value(kernel), _filetime_value(user))
|
|
408
|
+
if prev is None:
|
|
409
|
+
return 0.0, current
|
|
410
|
+
di, dk, du = (current[i] - prev[i] for i in range(3))
|
|
411
|
+
total = dk + du
|
|
412
|
+
busy = 0.0 if total <= 0 else (total - di) / total * 100
|
|
413
|
+
return max(0.0, min(100.0, busy)), current
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def free_memory_mb() -> int:
|
|
417
|
+
status = MEMORYSTATUSEX()
|
|
418
|
+
status.dwLength = ctypes.sizeof(status)
|
|
419
|
+
if not kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
|
|
420
|
+
raise _win_error("GlobalMemoryStatusEx")
|
|
421
|
+
return int(status.ullAvailPhys // (1024 * 1024))
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _open_process(pid: int, access: int = PROCESS_QUERY_LIMITED_INFORMATION) -> int | None:
|
|
425
|
+
handle = kernel32.OpenProcess(access, False, pid)
|
|
426
|
+
return int(handle) if handle else None
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _image_for_handle(handle: int) -> str | None:
|
|
430
|
+
size = wintypes.DWORD(32768)
|
|
431
|
+
buffer = ctypes.create_unicode_buffer(size.value)
|
|
432
|
+
if not kernel32.QueryFullProcessImageNameW(handle, 0, buffer, ctypes.byref(size)):
|
|
433
|
+
return None
|
|
434
|
+
return Path(buffer.value).name.lower()
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _governor_for_handle(handle: int, exe: str) -> bool:
|
|
438
|
+
if exe == "caddis_governor.exe":
|
|
439
|
+
return True
|
|
440
|
+
if not exe.startswith(("python", "pypy")):
|
|
441
|
+
return False
|
|
442
|
+
# ProcessCommandLineInformation returns a UNICODE_STRING backed by our buffer.
|
|
443
|
+
# Treat an unreadable Python command line as a possible job owner.
|
|
444
|
+
size = wintypes.ULONG()
|
|
445
|
+
ntdll.NtQueryInformationProcess(handle, 60, None, 0, ctypes.byref(size))
|
|
446
|
+
if not ctypes.sizeof(UNICODE_STRING) <= size.value <= 131072:
|
|
447
|
+
return True
|
|
448
|
+
buffer = ctypes.create_string_buffer(size.value)
|
|
449
|
+
if ntdll.NtQueryInformationProcess(handle, 60, buffer, len(buffer), ctypes.byref(size)) < 0:
|
|
450
|
+
return True
|
|
451
|
+
value = UNICODE_STRING.from_buffer(buffer)
|
|
452
|
+
start, end = ctypes.addressof(buffer), ctypes.addressof(buffer) + len(buffer)
|
|
453
|
+
if (not value.Buffer or value.Length % 2 or value.Buffer < start
|
|
454
|
+
or value.Buffer + value.Length > end):
|
|
455
|
+
return True
|
|
456
|
+
command = ctypes.wstring_at(value.Buffer, value.Length // 2).lower()
|
|
457
|
+
return "caddis_governor" in command
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def snapshot() -> dict[int, Proc]:
|
|
461
|
+
"""Return the current process table. An unreadable process has creation time zero."""
|
|
462
|
+
# Convert Unix nanoseconds to the Windows FILETIME epoch before acquiring ancestry.
|
|
463
|
+
started = time.time_ns() // 100 + 116444736000000000
|
|
464
|
+
handle = kernel32.CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0)
|
|
465
|
+
if handle == INVALID_HANDLE_VALUE:
|
|
466
|
+
raise _win_error("CreateToolhelp32Snapshot")
|
|
467
|
+
procs: dict[int, Proc] = {}
|
|
468
|
+
try:
|
|
469
|
+
entry = PROCESSENTRY32W()
|
|
470
|
+
entry.dwSize = ctypes.sizeof(entry)
|
|
471
|
+
more = kernel32.Process32FirstW(handle, ctypes.byref(entry))
|
|
472
|
+
if not more and ctypes.get_last_error() != ERROR_NO_MORE_FILES:
|
|
473
|
+
raise _win_error("Process32FirstW")
|
|
474
|
+
while more:
|
|
475
|
+
pid = int(entry.th32ProcessID)
|
|
476
|
+
exe = Path(entry.szExeFile).name.lower()
|
|
477
|
+
created = 0
|
|
478
|
+
governor = exe.startswith(("python", "pypy")) or exe == "caddis_governor.exe"
|
|
479
|
+
process = _open_process(pid)
|
|
480
|
+
if process is not None:
|
|
481
|
+
try:
|
|
482
|
+
times = _times_for_handle(process)
|
|
483
|
+
image = _image_for_handle(process)
|
|
484
|
+
if times is not None and 0 < times[0] <= started and image == exe:
|
|
485
|
+
created = times[0]
|
|
486
|
+
governor = _governor_for_handle(process, image)
|
|
487
|
+
finally:
|
|
488
|
+
kernel32.CloseHandle(process)
|
|
489
|
+
procs[pid] = Proc(pid, int(entry.th32ParentProcessID), exe, created, governor)
|
|
490
|
+
more = kernel32.Process32NextW(handle, ctypes.byref(entry))
|
|
491
|
+
if ctypes.get_last_error() not in (0, ERROR_NO_MORE_FILES):
|
|
492
|
+
raise _win_error("Process32NextW")
|
|
493
|
+
finally:
|
|
494
|
+
kernel32.CloseHandle(handle)
|
|
495
|
+
return procs
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def process_cpu_seconds(pids) -> dict[int, float]:
|
|
499
|
+
values: dict[int, float] = {}
|
|
500
|
+
for pid in pids:
|
|
501
|
+
handle = _open_process(int(pid))
|
|
502
|
+
if handle is None:
|
|
503
|
+
continue
|
|
504
|
+
try:
|
|
505
|
+
times = _times_for_handle(handle)
|
|
506
|
+
if times is not None:
|
|
507
|
+
values[int(pid)] = times[1]
|
|
508
|
+
finally:
|
|
509
|
+
kernel32.CloseHandle(handle)
|
|
510
|
+
return values
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def process_memory_mb(pids) -> dict[int, int]:
|
|
514
|
+
"""Return each readable process's resident working set in MiB."""
|
|
515
|
+
values: dict[int, int] = {}
|
|
516
|
+
for pid in pids:
|
|
517
|
+
handle = _open_process(int(pid), PROCESS_QUERY_LIMITED_INFORMATION | PROCESS_VM_READ)
|
|
518
|
+
if handle is None:
|
|
519
|
+
continue
|
|
520
|
+
try:
|
|
521
|
+
counters = PROCESS_MEMORY_COUNTERS()
|
|
522
|
+
counters.cb = ctypes.sizeof(counters)
|
|
523
|
+
if psapi.GetProcessMemoryInfo(handle, ctypes.byref(counters), counters.cb):
|
|
524
|
+
values[int(pid)] = int(counters.WorkingSetSize // (1024 * 1024))
|
|
525
|
+
finally:
|
|
526
|
+
kernel32.CloseHandle(handle)
|
|
527
|
+
return values
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def terminate_tree(pid: int, lineage: Lineage, procs: dict[int, Proc]) -> list[int]:
|
|
531
|
+
"""End a verified subtree, leaves first; hold identities through the whole operation."""
|
|
532
|
+
target = procs.get(pid)
|
|
533
|
+
if (target is None or target.created == 0 or lineage.marked.get(pid) != target.created
|
|
534
|
+
or pid not in lineage.verified or pid in protected_ancestors(procs, lineage.root_images)):
|
|
535
|
+
return []
|
|
536
|
+
|
|
537
|
+
children: dict[int, list[int]] = {}
|
|
538
|
+
for proc in procs.values():
|
|
539
|
+
parent = procs.get(proc.ppid)
|
|
540
|
+
if (parent is not None and (not parent.created or not proc.created
|
|
541
|
+
or parent.created <= proc.created)):
|
|
542
|
+
children.setdefault(parent.pid, []).append(proc.pid)
|
|
543
|
+
|
|
544
|
+
order: list[int] = []
|
|
545
|
+
seen: set[int] = set()
|
|
546
|
+
|
|
547
|
+
def visit(current: int) -> None:
|
|
548
|
+
if current in seen:
|
|
549
|
+
return
|
|
550
|
+
seen.add(current)
|
|
551
|
+
for child in children.get(current, []):
|
|
552
|
+
visit(child)
|
|
553
|
+
order.append(current)
|
|
554
|
+
|
|
555
|
+
visit(pid)
|
|
556
|
+
ended: list[int] = []
|
|
557
|
+
handles: dict[int, int] = {}
|
|
558
|
+
try:
|
|
559
|
+
# Check the entire subtree before ending any member. Unknown or reused identities
|
|
560
|
+
# cannot establish that this is still safe, especially for a job owner.
|
|
561
|
+
for current in order:
|
|
562
|
+
proc = procs[current]
|
|
563
|
+
if (not proc.created or current not in lineage.verified
|
|
564
|
+
or lineage.marked.get(current) != proc.created):
|
|
565
|
+
return []
|
|
566
|
+
handle = _open_process(current, PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION)
|
|
567
|
+
if handle is None:
|
|
568
|
+
return []
|
|
569
|
+
handles[current] = handle
|
|
570
|
+
times = _times_for_handle(handle)
|
|
571
|
+
image = _image_for_handle(handle)
|
|
572
|
+
if (times is None or times[0] != proc.created or image != proc.exe
|
|
573
|
+
or image in lineage.root_images or _governor_for_handle(handle, image)):
|
|
574
|
+
return []
|
|
575
|
+
for current in order:
|
|
576
|
+
if kernel32.TerminateProcess(handles[current], 1):
|
|
577
|
+
ended.append(current)
|
|
578
|
+
finally:
|
|
579
|
+
for handle in handles.values():
|
|
580
|
+
kernel32.CloseHandle(handle)
|
|
581
|
+
return ended
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _sample(prev_cpu: tuple[int, int, int] | None, lineage: Lineage,
|
|
585
|
+
cfg: GuardConfig) -> tuple[Sample, tuple[int, int, int], dict[int, Proc]]:
|
|
586
|
+
procs = snapshot()
|
|
587
|
+
lineage.update(procs, cfg.root_images)
|
|
588
|
+
cpu, current_cpu = host_cpu_percent(prev_cpu)
|
|
589
|
+
sample = Sample(time.time(), cpu, free_memory_mb(), process_cpu_seconds(procs),
|
|
590
|
+
process_memory_mb(procs))
|
|
591
|
+
return sample, current_cpu, procs
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _track_until(deadline: float, lineage: Lineage, roots: tuple[str, ...]) -> None:
|
|
595
|
+
"""Refresh lineage at most two seconds apart until the next full sample is due."""
|
|
596
|
+
while True:
|
|
597
|
+
remaining = deadline - time.monotonic()
|
|
598
|
+
if remaining <= 0:
|
|
599
|
+
return
|
|
600
|
+
time.sleep(min(LINEAGE_POLL_SECONDS, remaining))
|
|
601
|
+
if deadline - time.monotonic() > 0:
|
|
602
|
+
lineage.update(snapshot(), roots)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def run(budget: Path | None = None, max_ticks: int | None = None) -> int:
|
|
606
|
+
if sys.platform != "win32":
|
|
607
|
+
print("[caddis-host-guard] Windows-only in this build")
|
|
608
|
+
return 0
|
|
609
|
+
cfg = load_guard_config(budget)
|
|
610
|
+
lineage = Lineage()
|
|
611
|
+
window: list[Sample] = []
|
|
612
|
+
previous_cpu = None
|
|
613
|
+
ticks = 0
|
|
614
|
+
while max_ticks is None or ticks < max_ticks:
|
|
615
|
+
sample, previous_cpu, procs = _sample(previous_cpu, lineage, cfg)
|
|
616
|
+
window.append(sample)
|
|
617
|
+
cutoff = sample.ts - cfg.sustain_seconds
|
|
618
|
+
while len(window) > 2 and window[1].ts <= cutoff:
|
|
619
|
+
window.pop(0) # retain one boundary sample so the window covers the full sustain period
|
|
620
|
+
actions = decide(window, lineage, procs, cfg, os.cpu_count() or 1)
|
|
621
|
+
for action in actions:
|
|
622
|
+
ended = (terminate_tree(action.pid, lineage, procs)
|
|
623
|
+
if action.kind == "end" and action.pid is not None else [])
|
|
624
|
+
kind = "end-failed" if action.kind == "end" and not ended else action.kind
|
|
625
|
+
record = {"ts": sample.ts, "host_cpu": sample.host_cpu, "free_mb": sample.free_mb,
|
|
626
|
+
"kind": kind, "pid": action.pid, "exe": action.exe,
|
|
627
|
+
"reason": action.reason, "killed": ended, "ended_count": len(ended)}
|
|
628
|
+
append_log(guard_log_path(), record)
|
|
629
|
+
print(json.dumps(record, sort_keys=True), flush=True)
|
|
630
|
+
ticks += 1
|
|
631
|
+
if max_ticks is not None and ticks >= max_ticks:
|
|
632
|
+
break
|
|
633
|
+
_track_until(time.monotonic() + cfg.sample_seconds, lineage, cfg.root_images)
|
|
634
|
+
return 0
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def _once(budget: Path | None = None) -> int:
|
|
638
|
+
if sys.platform != "win32":
|
|
639
|
+
print("[caddis-host-guard] Windows-only in this build")
|
|
640
|
+
return 0
|
|
641
|
+
cfg = load_guard_config(budget)
|
|
642
|
+
_, previous = host_cpu_percent(None)
|
|
643
|
+
time.sleep(1)
|
|
644
|
+
cpu, _ = host_cpu_percent(previous)
|
|
645
|
+
procs = snapshot()
|
|
646
|
+
lineage = Lineage()
|
|
647
|
+
lineage.update(procs, cfg.root_images)
|
|
648
|
+
print(json.dumps({"host_cpu": cpu, "free_mb": free_memory_mb(),
|
|
649
|
+
"agent_processes": len(lineage.marked)}, sort_keys=True))
|
|
650
|
+
return 0
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def _positive(value: str) -> int:
|
|
654
|
+
try:
|
|
655
|
+
number = int(value)
|
|
656
|
+
except ValueError as exc:
|
|
657
|
+
raise argparse.ArgumentTypeError("expected a positive integer") from exc
|
|
658
|
+
if number < 1:
|
|
659
|
+
raise argparse.ArgumentTypeError("expected a positive integer")
|
|
660
|
+
return number
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def main(argv: list[str] | None = None) -> int:
|
|
664
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
665
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
666
|
+
run_parser = commands.add_parser("run", help="monitor the host continuously")
|
|
667
|
+
run_parser.add_argument("--budget", type=Path)
|
|
668
|
+
run_parser.add_argument("--max-ticks", type=_positive)
|
|
669
|
+
once_parser = commands.add_parser("once", help="sample the host without taking action")
|
|
670
|
+
once_parser.add_argument("--budget", type=Path)
|
|
671
|
+
summary_parser = commands.add_parser("summary", help="report recent guard actions")
|
|
672
|
+
summary_parser.add_argument("--hours", type=_positive, default=24)
|
|
673
|
+
args = parser.parse_args(argv)
|
|
674
|
+
try:
|
|
675
|
+
if args.command == "run":
|
|
676
|
+
return run(args.budget, args.max_ticks)
|
|
677
|
+
if args.command == "once":
|
|
678
|
+
return _once(args.budget)
|
|
679
|
+
line = summarise(guard_log_path(), args.hours)
|
|
680
|
+
if line:
|
|
681
|
+
print(line)
|
|
682
|
+
return 0
|
|
683
|
+
except GuardError as exc:
|
|
684
|
+
print(f"[caddis-host-guard] ERROR: {exc}", file=sys.stderr)
|
|
685
|
+
return EXIT_CONFIG
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
if __name__ == "__main__":
|
|
689
|
+
raise SystemExit(main())
|