@caddis/cli 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundles/antigravity-plugin/guard_agy.py +62 -13
- package/bundles/antigravity-plugin/plugin.json +1 -1
- package/bundles/antigravity-plugin/repo_freshness.py +146 -0
- package/bundles/antigravity-plugin/scripts/caddis-run.ps1 +11 -0
- package/bundles/antigravity-plugin/scripts/caddis-run.sh +9 -0
- package/bundles/antigravity-plugin/scripts/caddis_gate.py +24 -3
- package/bundles/antigravity-plugin/scripts/caddis_governor.py +542 -0
- package/bundles/antigravity-plugin/scripts/caddis_host_guard.py +689 -0
- package/bundles/antigravity-plugin/scripts/caddis_inventory.py +7 -3
- package/bundles/antigravity-plugin/scripts/caddis_lanes.py +1632 -0
- package/bundles/antigravity-plugin/scripts/caddis_spawn.py +159 -14
- package/bundles/antigravity-plugin/scripts/check_doc_coverage.py +59 -7
- package/bundles/antigravity-plugin/scripts/install-host-guard.ps1 +393 -0
- package/bundles/antigravity-plugin/skills/feature-plan/SKILL.md +30 -4
- package/bundles/antigravity-plugin/skills/implement/SKILL.md +48 -1
- package/bundles/antigravity-plugin/skills/kb/SKILL.md +4 -0
- package/bundles/antigravity-plugin/skills/spawn-session/SKILL.md +89 -16
- package/bundles/antigravity-plugin/warm_start_agy.py +38 -4
- package/bundles/manifest.json +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
"""caddis_governor — run a command inside a capped Job Object so an agent can never saturate the host.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python caddis_governor.py exec --scope session|lane [--budget PATH] -- <command...>
|
|
5
|
+
python caddis_governor.py show [--budget PATH]
|
|
6
|
+
|
|
7
|
+
Why this exists: on 2026-09-13 a coding-agent lane ran 32 all-core busy loops for about 36 minutes
|
|
8
|
+
on a shared dev box that was also a desktop and the CI runners, freezing both. A prompt rule cannot
|
|
9
|
+
stop that. A Job Object can: it caps CPU, memory and process count for a process and every process
|
|
10
|
+
it starts through normal process creation, whatever the prompt says.
|
|
11
|
+
|
|
12
|
+
How a process enters the job without a race: the governor assigns ITSELF to the job, then starts
|
|
13
|
+
the command. Every normally-created descendant inherits the job. Killing the governor ends its
|
|
14
|
+
handle to the job, and KILL_ON_JOB_CLOSE then kills the whole tree. After setup the governor keeps
|
|
15
|
+
only a query-only handle, and the job's DACL grants only query access to its owner, so a same-user
|
|
16
|
+
child can neither duplicate a writable handle nor open the job writable by name.
|
|
17
|
+
|
|
18
|
+
THREAT MODEL: an agent that ACCIDENTALLY or naively consumes resources (it was told to load the
|
|
19
|
+
box, or it "fixes" slow tests by raising a cap). The limits hold against that. They do NOT hold
|
|
20
|
+
against an agent deliberately escaping the job. Known escapes, documented rather than hidden:
|
|
21
|
+
* a same-user process creating a child with a spoofed parent (PROC_THREAD_ATTRIBUTE_PARENT_PROCESS)
|
|
22
|
+
or through out-of-process COM / shell activation (the child lands outside the job);
|
|
23
|
+
* processes created by WMI, Task Scheduler, a service, or elevation;
|
|
24
|
+
* work done inside an already-running server (a database, a model server, a container daemon).
|
|
25
|
+
The real fix for a hostile agent is running it under a separate low-privilege account.
|
|
26
|
+
|
|
27
|
+
Limits come from the user budget (~/.caddis/host-budget.toml) CLAMPED by the machine budget
|
|
28
|
+
(%ProgramData%\\caddis\\host-budget.toml on Windows, /etc/caddis/host-budget.toml elsewhere), which only
|
|
29
|
+
administrators can write. An agent can lower its own limits; it cannot raise them past the machine
|
|
30
|
+
ceiling. With no machine budget file, the ceiling is the built-in defaults, so the user budget can
|
|
31
|
+
only lower them.
|
|
32
|
+
|
|
33
|
+
On Linux the limits are applied with `systemd-run --user --scope`. Where that is unavailable (and on
|
|
34
|
+
macOS) the governor refuses (exit 3) unless `--allow-uncapped` is passed explicitly, which runs the
|
|
35
|
+
command at lower priority with a warning and never marks it as governed.
|
|
36
|
+
|
|
37
|
+
Nested jobs MULTIPLY CPU caps (measured 2026-09-14): an inner job's rate is a share of its parent's
|
|
38
|
+
allowance. So a lane inside a governed session asks for a rate relative to the session's cap. The
|
|
39
|
+
session is identified by a NAMED job (CADDIS_GOVERNED_JOB). The inner governor opens it for query,
|
|
40
|
+
proves it is a member, and reads the job's own CPU rate; the environment's percent is informational
|
|
41
|
+
only. (QueryInformationJobObject with a NULL handle is NOT a reliable way
|
|
42
|
+
to find the immediate job: measured, it returned an outer job's limits.)
|
|
43
|
+
|
|
44
|
+
Refuses to run the command at all when the limits cannot be applied (exit 3). Degrading open here
|
|
45
|
+
would be the exact failure this file exists to prevent.
|
|
46
|
+
"""
|
|
47
|
+
from __future__ import annotations
|
|
48
|
+
|
|
49
|
+
import argparse
|
|
50
|
+
import os
|
|
51
|
+
import secrets
|
|
52
|
+
import shutil
|
|
53
|
+
import subprocess
|
|
54
|
+
import sys
|
|
55
|
+
from dataclasses import dataclass
|
|
56
|
+
from pathlib import Path
|
|
57
|
+
|
|
58
|
+
EXIT_USAGE = 2
|
|
59
|
+
EXIT_CONFIG = 3
|
|
60
|
+
SCOPES = ("session", "lane")
|
|
61
|
+
MAX_CPU_PERCENT = 100
|
|
62
|
+
MAX_MEMORY_MB = 1_048_576
|
|
63
|
+
MAX_PROCESSES = 65_535
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class GovernorError(Exception):
|
|
67
|
+
"""The limits could not be applied; the command must not run ungoverned."""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass(frozen=True)
|
|
71
|
+
class JobLimits:
|
|
72
|
+
cpu_percent: int
|
|
73
|
+
memory_mb: int
|
|
74
|
+
max_processes: int
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
DEFAULT_BUDGET = {
|
|
78
|
+
"session": JobLimits(50, 24576, 256),
|
|
79
|
+
"lane": JobLimits(25, 8192, 64),
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if sys.platform == "win32":
|
|
83
|
+
import ctypes
|
|
84
|
+
from ctypes import wintypes
|
|
85
|
+
|
|
86
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
87
|
+
shell32 = ctypes.WinDLL("shell32", use_last_error=True)
|
|
88
|
+
advapi32 = ctypes.WinDLL("advapi32", use_last_error=True)
|
|
89
|
+
|
|
90
|
+
class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
|
|
91
|
+
_fields_ = [("PerProcessUserTimeLimit", ctypes.c_int64), ("PerJobUserTimeLimit", ctypes.c_int64),
|
|
92
|
+
("LimitFlags", wintypes.DWORD), ("MinimumWorkingSetSize", ctypes.c_size_t),
|
|
93
|
+
("MaximumWorkingSetSize", ctypes.c_size_t), ("ActiveProcessLimit", wintypes.DWORD),
|
|
94
|
+
("Affinity", ctypes.c_size_t), ("PriorityClass", wintypes.DWORD),
|
|
95
|
+
("SchedulingClass", wintypes.DWORD)]
|
|
96
|
+
|
|
97
|
+
class IO_COUNTERS(ctypes.Structure):
|
|
98
|
+
_fields_ = [(n, ctypes.c_uint64) for n in ("ReadOperationCount", "WriteOperationCount",
|
|
99
|
+
"OtherOperationCount", "ReadTransferCount",
|
|
100
|
+
"WriteTransferCount", "OtherTransferCount")]
|
|
101
|
+
|
|
102
|
+
class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
|
|
103
|
+
_fields_ = [("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION), ("IoInfo", IO_COUNTERS),
|
|
104
|
+
("ProcessMemoryLimit", ctypes.c_size_t), ("JobMemoryLimit", ctypes.c_size_t),
|
|
105
|
+
("PeakProcessMemoryUsed", ctypes.c_size_t), ("PeakJobMemoryUsed", ctypes.c_size_t)]
|
|
106
|
+
|
|
107
|
+
class JOBOBJECT_CPU_RATE_CONTROL_INFORMATION(ctypes.Structure):
|
|
108
|
+
_fields_ = [("ControlFlags", wintypes.DWORD), ("CpuRate", wintypes.DWORD)]
|
|
109
|
+
|
|
110
|
+
class JOBOBJECT_BASIC_ACCOUNTING_INFORMATION(ctypes.Structure):
|
|
111
|
+
_fields_ = [("TotalUserTime", ctypes.c_int64), ("TotalKernelTime", ctypes.c_int64),
|
|
112
|
+
("ThisPeriodTotalUserTime", ctypes.c_int64), ("ThisPeriodTotalKernelTime", ctypes.c_int64),
|
|
113
|
+
("TotalPageFaultCount", wintypes.DWORD), ("TotalProcesses", wintypes.DWORD),
|
|
114
|
+
("ActiveProcesses", wintypes.DWORD), ("TotalTerminatedProcesses", wintypes.DWORD)]
|
|
115
|
+
|
|
116
|
+
class JOBOBJECT_BASIC_PROCESS_ID_LIST(ctypes.Structure):
|
|
117
|
+
_fields_ = [("NumberOfAssignedProcesses", wintypes.DWORD), ("NumberOfProcessIdsInList", wintypes.DWORD),
|
|
118
|
+
("ProcessIdList", ctypes.c_size_t * 1024)]
|
|
119
|
+
|
|
120
|
+
class SECURITY_ATTRIBUTES(ctypes.Structure):
|
|
121
|
+
_fields_ = [("nLength", wintypes.DWORD), ("lpSecurityDescriptor", ctypes.c_void_p),
|
|
122
|
+
("bInheritHandle", wintypes.BOOL)]
|
|
123
|
+
|
|
124
|
+
JobObjectBasicAccountingInformation = 1
|
|
125
|
+
JobObjectBasicProcessIdList = 3
|
|
126
|
+
JobObjectExtendedLimitInformation = 9
|
|
127
|
+
JobObjectCpuRateControlInformation = 15
|
|
128
|
+
JOB_OBJECT_LIMIT_ACTIVE_PROCESS = 0x8
|
|
129
|
+
JOB_OBJECT_LIMIT_PRIORITY_CLASS = 0x20
|
|
130
|
+
JOB_OBJECT_LIMIT_JOB_MEMORY = 0x200
|
|
131
|
+
JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x800 # never set: a child could leave the job
|
|
132
|
+
JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK = 0x1000 # never set: same, silently
|
|
133
|
+
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
|
|
134
|
+
JOB_OBJECT_CPU_RATE_CONTROL_ENABLE = 0x1
|
|
135
|
+
JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP = 0x4
|
|
136
|
+
BELOW_NORMAL_PRIORITY_CLASS = 0x4000
|
|
137
|
+
JOB_OBJECT_QUERY = 0x0004
|
|
138
|
+
ERROR_ALREADY_EXISTS = 183
|
|
139
|
+
CSIDL_COMMON_APPDATA = 0x23
|
|
140
|
+
SDDL_REVISION_1 = 1
|
|
141
|
+
# SYSTEM and Administrators: full. OWNER RIGHTS and Everyone: query only. The explicit OWNER RIGHTS
|
|
142
|
+
# ACE removes the owner's implicit WRITE_DAC, so the creating user cannot grant itself write access.
|
|
143
|
+
JOB_SDDL = "D:P(A;;GA;;;SY)(A;;GA;;;BA)(A;;0x4;;;OW)(A;;0x4;;;WD)"
|
|
144
|
+
|
|
145
|
+
kernel32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR]
|
|
146
|
+
kernel32.CreateJobObjectW.restype = wintypes.HANDLE
|
|
147
|
+
kernel32.OpenJobObjectW.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.LPCWSTR]
|
|
148
|
+
kernel32.OpenJobObjectW.restype = wintypes.HANDLE
|
|
149
|
+
kernel32.SetInformationJobObject.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
|
|
150
|
+
kernel32.SetInformationJobObject.restype = wintypes.BOOL
|
|
151
|
+
kernel32.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE]
|
|
152
|
+
kernel32.AssignProcessToJobObject.restype = wintypes.BOOL
|
|
153
|
+
kernel32.QueryInformationJobObject.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p,
|
|
154
|
+
wintypes.DWORD, ctypes.POINTER(wintypes.DWORD)]
|
|
155
|
+
kernel32.QueryInformationJobObject.restype = wintypes.BOOL
|
|
156
|
+
kernel32.IsProcessInJob.argtypes = [wintypes.HANDLE, wintypes.HANDLE, ctypes.POINTER(wintypes.BOOL)]
|
|
157
|
+
kernel32.IsProcessInJob.restype = wintypes.BOOL
|
|
158
|
+
kernel32.DuplicateHandle.argtypes = [wintypes.HANDLE, wintypes.HANDLE, wintypes.HANDLE,
|
|
159
|
+
ctypes.POINTER(wintypes.HANDLE), wintypes.DWORD, wintypes.BOOL,
|
|
160
|
+
wintypes.DWORD]
|
|
161
|
+
kernel32.DuplicateHandle.restype = wintypes.BOOL
|
|
162
|
+
kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
|
|
163
|
+
kernel32.CloseHandle.restype = wintypes.BOOL
|
|
164
|
+
kernel32.GetCurrentProcess.argtypes = []
|
|
165
|
+
kernel32.GetCurrentProcess.restype = wintypes.HANDLE
|
|
166
|
+
kernel32.LocalFree.argtypes = [ctypes.c_void_p]
|
|
167
|
+
kernel32.LocalFree.restype = ctypes.c_void_p
|
|
168
|
+
advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW.argtypes = [
|
|
169
|
+
wintypes.LPCWSTR, wintypes.DWORD, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(wintypes.ULONG)]
|
|
170
|
+
advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW.restype = wintypes.BOOL
|
|
171
|
+
shell32.SHGetFolderPathW.argtypes = [wintypes.HWND, ctypes.c_int, wintypes.HANDLE, wintypes.DWORD,
|
|
172
|
+
wintypes.LPWSTR]
|
|
173
|
+
shell32.SHGetFolderPathW.restype = ctypes.c_long
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
# ── limits ──────────────────────────────────────────────────────────────────
|
|
177
|
+
|
|
178
|
+
def budget_path() -> Path:
|
|
179
|
+
"""The user budget. Agent-writable, so it can only LOWER limits below the machine ceiling."""
|
|
180
|
+
return Path(os.environ.get("CADDIS_HOME") or Path.home()) / ".caddis" / "host-budget.toml"
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def machine_budget_path() -> Path:
|
|
184
|
+
"""The admin-owned ceiling. Resolved without environment variables an agent could persist."""
|
|
185
|
+
if sys.platform == "win32":
|
|
186
|
+
buf = ctypes.create_unicode_buffer(260)
|
|
187
|
+
if shell32.SHGetFolderPathW(None, CSIDL_COMMON_APPDATA, None, 0, buf) == 0 and buf.value:
|
|
188
|
+
return Path(buf.value) / "caddis" / "host-budget.toml"
|
|
189
|
+
return Path(r"C:\ProgramData") / "caddis" / "host-budget.toml"
|
|
190
|
+
return Path("/etc/caddis/host-budget.toml")
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _as_int(scope: str, key: str, value: object) -> int:
|
|
194
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
195
|
+
raise GovernorError(f"invalid [{scope}] {key}: {value!r}")
|
|
196
|
+
return value
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _read_table(file: Path, scope: str) -> dict:
|
|
200
|
+
try:
|
|
201
|
+
import tomllib
|
|
202
|
+
except ImportError as exc:
|
|
203
|
+
# An existing budget that cannot be read must not silently become the (looser) defaults.
|
|
204
|
+
raise GovernorError(f"{file} exists but this Python has no tomllib (need 3.11+)") from exc
|
|
205
|
+
try:
|
|
206
|
+
data = tomllib.loads(file.read_text(encoding="utf-8"))
|
|
207
|
+
except Exception as exc: # a broken budget must never silently fall back to defaults
|
|
208
|
+
raise GovernorError(f"cannot parse {file}: {exc}") from exc
|
|
209
|
+
table = data.get(scope, {})
|
|
210
|
+
if not isinstance(table, dict):
|
|
211
|
+
raise GovernorError(f"[{scope}] in {file} is not a table")
|
|
212
|
+
return table
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _validated(scope: str, table: dict, base: JobLimits) -> JobLimits:
|
|
216
|
+
values = {
|
|
217
|
+
"cpu_percent": _as_int(scope, "cpu_percent", table.get("cpu_percent", base.cpu_percent)),
|
|
218
|
+
"memory_mb": _as_int(scope, "memory_mb", table.get("memory_mb", base.memory_mb)),
|
|
219
|
+
"max_processes": _as_int(scope, "max_processes", table.get("max_processes", base.max_processes)),
|
|
220
|
+
}
|
|
221
|
+
if not 1 <= values["cpu_percent"] <= MAX_CPU_PERCENT:
|
|
222
|
+
raise GovernorError(f"invalid [{scope}] cpu_percent: {values['cpu_percent']}")
|
|
223
|
+
if not 256 <= values["memory_mb"] <= MAX_MEMORY_MB:
|
|
224
|
+
raise GovernorError(f"invalid [{scope}] memory_mb: {values['memory_mb']}")
|
|
225
|
+
if not 4 <= values["max_processes"] <= MAX_PROCESSES:
|
|
226
|
+
raise GovernorError(f"invalid [{scope}] max_processes: {values['max_processes']}")
|
|
227
|
+
return JobLimits(**values)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def load_limits(scope: str, path: Path | None = None, machine_path: Path | None = None) -> JobLimits:
|
|
231
|
+
"""The limits for `scope`: the user budget (or defaults), clamped by the machine ceiling.
|
|
232
|
+
|
|
233
|
+
With no machine budget file the defaults are the ceiling. A broken or unreadable file is an error.
|
|
234
|
+
An optional machine_path adds a stricter ceiling; it never replaces the real machine budget.
|
|
235
|
+
"""
|
|
236
|
+
if scope not in SCOPES:
|
|
237
|
+
raise GovernorError(f"unknown scope: {scope!r} (expected one of {', '.join(SCOPES)})")
|
|
238
|
+
user_file = path or budget_path()
|
|
239
|
+
limits = DEFAULT_BUDGET[scope]
|
|
240
|
+
if user_file.is_file():
|
|
241
|
+
limits = _validated(scope, _read_table(user_file, scope), limits)
|
|
242
|
+
ceiling_file = machine_budget_path()
|
|
243
|
+
# No admin ceiling: the defaults ARE the ceiling, so an agent-writable budget can only lower them.
|
|
244
|
+
ceiling = DEFAULT_BUDGET[scope]
|
|
245
|
+
if ceiling_file.is_file():
|
|
246
|
+
# Keys the admin file does not set keep the DEFAULT ceiling: a partial or empty admin file
|
|
247
|
+
# must never be weaker than no file at all.
|
|
248
|
+
ceiling = _validated(scope, _read_table(ceiling_file, scope), DEFAULT_BUDGET[scope])
|
|
249
|
+
if machine_path is not None and machine_path.is_file():
|
|
250
|
+
requested = _validated(scope, _read_table(machine_path, scope), ceiling)
|
|
251
|
+
ceiling = JobLimits(min(ceiling.cpu_percent, requested.cpu_percent),
|
|
252
|
+
min(ceiling.memory_mb, requested.memory_mb),
|
|
253
|
+
min(ceiling.max_processes, requested.max_processes))
|
|
254
|
+
return JobLimits(min(limits.cpu_percent, ceiling.cpu_percent),
|
|
255
|
+
min(limits.memory_mb, ceiling.memory_mb),
|
|
256
|
+
min(limits.max_processes, ceiling.max_processes))
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
# ── nesting ─────────────────────────────────────────────────────────────────
|
|
260
|
+
|
|
261
|
+
def verified_parent_percent(env: dict | None = None) -> int | None:
|
|
262
|
+
"""The enclosing governed job's machine percent, or None when it cannot be PROVEN (Windows).
|
|
263
|
+
|
|
264
|
+
Opens the job named by CADDIS_GOVERNED_JOB for query, requires that this process is a member and
|
|
265
|
+
that the job has CPU hard-cap control, and reads the job's own CpuRate (rounded up). The
|
|
266
|
+
environment's percent is informational only and never used here.
|
|
267
|
+
"""
|
|
268
|
+
if sys.platform != "win32":
|
|
269
|
+
return None
|
|
270
|
+
e = os.environ if env is None else env
|
|
271
|
+
name = str(e.get("CADDIS_GOVERNED_JOB", "")).strip()
|
|
272
|
+
if not name:
|
|
273
|
+
return None
|
|
274
|
+
job = kernel32.OpenJobObjectW(JOB_OBJECT_QUERY, False, name)
|
|
275
|
+
if not job:
|
|
276
|
+
return None
|
|
277
|
+
try:
|
|
278
|
+
member = wintypes.BOOL()
|
|
279
|
+
if not kernel32.IsProcessInJob(kernel32.GetCurrentProcess(), job, ctypes.byref(member)) or not member.value:
|
|
280
|
+
return None
|
|
281
|
+
cr = JOBOBJECT_CPU_RATE_CONTROL_INFORMATION()
|
|
282
|
+
if not kernel32.QueryInformationJobObject(job, JobObjectCpuRateControlInformation, ctypes.byref(cr),
|
|
283
|
+
ctypes.sizeof(cr), None):
|
|
284
|
+
return None
|
|
285
|
+
want = JOB_OBJECT_CPU_RATE_CONTROL_ENABLE | JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP
|
|
286
|
+
if cr.ControlFlags & want != want or cr.CpuRate < 1:
|
|
287
|
+
return None
|
|
288
|
+
# A nested job's CpuRate is a share of ITS parent, so it bounds the machine percent from above.
|
|
289
|
+
# The environment's percent is deliberately NOT used: this value is a divisor, so a lower
|
|
290
|
+
# number would widen the lane's cap. Round up for the same reason.
|
|
291
|
+
return max(1, (cr.CpuRate + 99) // 100)
|
|
292
|
+
finally:
|
|
293
|
+
kernel32.CloseHandle(job)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def effective_cpu_rate(limits: JobLimits, parent_percent: int | None) -> int:
|
|
297
|
+
"""CpuRate for the job: percent x 100, relative to an enclosing governed job when nested.
|
|
298
|
+
|
|
299
|
+
Nested job caps multiply, so a lane inside a 50% session asks for 25/50 of the session's
|
|
300
|
+
allowance to end up with 25% of the machine. With no verified parent the rate is absolute.
|
|
301
|
+
"""
|
|
302
|
+
if parent_percent is None:
|
|
303
|
+
return limits.cpu_percent * 100
|
|
304
|
+
return max(1, min(10000, round(limits.cpu_percent * 10000 / parent_percent)))
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def effective_machine_percent(limits: JobLimits, parent_percent: int | None) -> int:
|
|
308
|
+
return limits.cpu_percent if parent_percent is None else min(limits.cpu_percent, parent_percent)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
# ── Windows Job Objects ─────────────────────────────────────────────────────
|
|
312
|
+
|
|
313
|
+
def _winerr(call: str) -> GovernorError:
|
|
314
|
+
return GovernorError(f"{call} failed: winerror {ctypes.get_last_error()}")
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def new_job_name() -> str:
|
|
318
|
+
return f"Local\\caddis-governor-{os.getpid()}-{secrets.token_hex(8)}"
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def create_job(limits: JobLimits, cpu_rate: int | None = None, name: str | None = None) -> int:
|
|
322
|
+
"""A new, non-inheritable, full-access job handle with every limit applied. Windows only.
|
|
323
|
+
|
|
324
|
+
A named job gets a DACL that grants its owner query access only, and refuses a name that exists.
|
|
325
|
+
"""
|
|
326
|
+
sd = ctypes.c_void_p()
|
|
327
|
+
sa = None
|
|
328
|
+
if name is not None:
|
|
329
|
+
if not advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW(JOB_SDDL, SDDL_REVISION_1,
|
|
330
|
+
ctypes.byref(sd), None):
|
|
331
|
+
raise _winerr("ConvertStringSecurityDescriptorToSecurityDescriptorW")
|
|
332
|
+
sa = SECURITY_ATTRIBUTES(ctypes.sizeof(SECURITY_ATTRIBUTES), sd, False)
|
|
333
|
+
try:
|
|
334
|
+
job = kernel32.CreateJobObjectW(ctypes.byref(sa) if sa is not None else None, name)
|
|
335
|
+
err = ctypes.get_last_error() # read before LocalFree can overwrite it
|
|
336
|
+
finally:
|
|
337
|
+
if sd:
|
|
338
|
+
kernel32.LocalFree(sd)
|
|
339
|
+
already = err == ERROR_ALREADY_EXISTS
|
|
340
|
+
if not job:
|
|
341
|
+
raise GovernorError(f"CreateJobObjectW failed: winerror {err}")
|
|
342
|
+
try:
|
|
343
|
+
if name is not None and already:
|
|
344
|
+
raise GovernorError(f"a job named {name} already exists; refusing to join a job we did not create")
|
|
345
|
+
flags = (JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_ACTIVE_PROCESS
|
|
346
|
+
| JOB_OBJECT_LIMIT_PRIORITY_CLASS | JOB_OBJECT_LIMIT_JOB_MEMORY)
|
|
347
|
+
if flags & (JOB_OBJECT_LIMIT_BREAKAWAY_OK | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK):
|
|
348
|
+
raise GovernorError("breakaway flags must never be set")
|
|
349
|
+
ext = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
|
|
350
|
+
ext.BasicLimitInformation.LimitFlags = flags
|
|
351
|
+
ext.BasicLimitInformation.ActiveProcessLimit = limits.max_processes
|
|
352
|
+
ext.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS
|
|
353
|
+
ext.JobMemoryLimit = limits.memory_mb * 1024 * 1024
|
|
354
|
+
if not kernel32.SetInformationJobObject(job, JobObjectExtendedLimitInformation,
|
|
355
|
+
ctypes.byref(ext), ctypes.sizeof(ext)):
|
|
356
|
+
raise _winerr("SetInformationJobObject(extended limits)")
|
|
357
|
+
rate = limits.cpu_percent * 100 if cpu_rate is None else cpu_rate
|
|
358
|
+
cr = JOBOBJECT_CPU_RATE_CONTROL_INFORMATION(
|
|
359
|
+
JOB_OBJECT_CPU_RATE_CONTROL_ENABLE | JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP, rate)
|
|
360
|
+
if not kernel32.SetInformationJobObject(job, JobObjectCpuRateControlInformation,
|
|
361
|
+
ctypes.byref(cr), ctypes.sizeof(cr)):
|
|
362
|
+
raise _winerr("SetInformationJobObject(cpu rate)")
|
|
363
|
+
except BaseException:
|
|
364
|
+
kernel32.CloseHandle(job)
|
|
365
|
+
raise
|
|
366
|
+
return job
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def assign_current_process(job: int) -> None:
|
|
370
|
+
if not kernel32.AssignProcessToJobObject(job, kernel32.GetCurrentProcess()):
|
|
371
|
+
raise _winerr("AssignProcessToJobObject")
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def reduce_to_query_handle(job: int) -> int:
|
|
375
|
+
"""Swap the full-access handle for a JOB_OBJECT_QUERY-only duplicate and close the original.
|
|
376
|
+
|
|
377
|
+
A same-user child can duplicate any handle the governor holds. After this, the only handle it
|
|
378
|
+
can reach cannot change limits. The query handle keeps the job alive, so KILL_ON_JOB_CLOSE still
|
|
379
|
+
fires when the governor exits.
|
|
380
|
+
"""
|
|
381
|
+
me = kernel32.GetCurrentProcess()
|
|
382
|
+
query = wintypes.HANDLE()
|
|
383
|
+
if not kernel32.DuplicateHandle(me, job, me, ctypes.byref(query), JOB_OBJECT_QUERY, False, 0):
|
|
384
|
+
raise _winerr("DuplicateHandle(JOB_OBJECT_QUERY)")
|
|
385
|
+
kernel32.CloseHandle(job)
|
|
386
|
+
return query.value
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def job_cpu_seconds(job: int) -> float:
|
|
390
|
+
acc = JOBOBJECT_BASIC_ACCOUNTING_INFORMATION()
|
|
391
|
+
if not kernel32.QueryInformationJobObject(job, JobObjectBasicAccountingInformation, ctypes.byref(acc),
|
|
392
|
+
ctypes.sizeof(acc), None):
|
|
393
|
+
raise _winerr("QueryInformationJobObject(accounting)")
|
|
394
|
+
return (acc.TotalUserTime + acc.TotalKernelTime) / 1e7
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def job_process_ids(job: int) -> list[int]:
|
|
398
|
+
"""Up to 1024 process ids in the job (the fixed buffer size)."""
|
|
399
|
+
lst = JOBOBJECT_BASIC_PROCESS_ID_LIST()
|
|
400
|
+
if not kernel32.QueryInformationJobObject(job, JobObjectBasicProcessIdList, ctypes.byref(lst),
|
|
401
|
+
ctypes.sizeof(lst), None):
|
|
402
|
+
raise _winerr("QueryInformationJobObject(process ids)")
|
|
403
|
+
return [int(lst.ProcessIdList[i]) for i in range(lst.NumberOfProcessIdsInList)]
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def close_job(job: int) -> None:
|
|
407
|
+
"""Close a job handle. Never call it on the last handle of a job the calling process belongs to."""
|
|
408
|
+
kernel32.CloseHandle(job)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
# ── other platforms ─────────────────────────────────────────────────────────
|
|
412
|
+
|
|
413
|
+
def posix_argv(argv: list[str], limits: JobLimits, which=shutil.which, cpu_count=os.cpu_count) -> list[str] | None:
|
|
414
|
+
"""systemd user scope limits. Weaker than a Job Object: user scopes do not nest (a lane scope is
|
|
415
|
+
a sibling of its session scope), an agent can call `systemd-run --user` itself, and CPUQuota is
|
|
416
|
+
silently ignored when the cpu controller is not delegated to the user manager. TasksMax counts
|
|
417
|
+
THREADS, not processes, so on Linux `max_processes` is a thread limit."""
|
|
418
|
+
if not which("systemd-run"):
|
|
419
|
+
return None
|
|
420
|
+
return ["systemd-run", "--user", "--scope", "--quiet",
|
|
421
|
+
"-p", f"CPUQuota={limits.cpu_percent * (cpu_count() or 1)}%",
|
|
422
|
+
"-p", f"MemoryMax={limits.memory_mb}M",
|
|
423
|
+
"-p", f"TasksMax={limits.max_processes}",
|
|
424
|
+
"--", *argv]
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
# ── exec ────────────────────────────────────────────────────────────────────
|
|
428
|
+
|
|
429
|
+
def _run_child(argv: list[str], env: dict) -> int:
|
|
430
|
+
try:
|
|
431
|
+
# CreateProcess does not search PATHEXT. Resolve npm-style .CMD shims before launch.
|
|
432
|
+
command = shutil.which(argv[0], path=env.get("PATH"))
|
|
433
|
+
if command is None:
|
|
434
|
+
raise FileNotFoundError(argv[0])
|
|
435
|
+
return subprocess.run([command, *argv[1:]], env=env).returncode
|
|
436
|
+
except FileNotFoundError:
|
|
437
|
+
sys.stderr.write(f"[caddis-governor] command not found: {argv[0]}\n")
|
|
438
|
+
return 127
|
|
439
|
+
except KeyboardInterrupt:
|
|
440
|
+
return 130
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _host_kind() -> str:
|
|
444
|
+
"""Return "windows" (native Python), "non-native-windows" (MSYS2/Cygwin on Windows) or "posix"."""
|
|
445
|
+
if sys.platform in ("cygwin", "msys") or (os.name == "nt" and sys.platform != "win32"):
|
|
446
|
+
return "non-native-windows"
|
|
447
|
+
return "windows" if sys.platform == "win32" else "posix"
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def exec_governed(argv: list[str], scope: str, budget: Path | None = None,
|
|
451
|
+
machine_budget: Path | None = None, allow_uncapped: bool = False) -> int:
|
|
452
|
+
"""Run `argv` under the scope's limits and return its exit code.
|
|
453
|
+
|
|
454
|
+
On Windows this puts the CURRENT process into the job for the rest of its life, so it is only
|
|
455
|
+
called from the `exec` entry point. The last job handle is never closed explicitly: the governor
|
|
456
|
+
is a member, so closing it would kill the governor before it returns the exit code. Process exit
|
|
457
|
+
closes it instead, which also kills any descendants the command left behind.
|
|
458
|
+
"""
|
|
459
|
+
try:
|
|
460
|
+
host = _host_kind()
|
|
461
|
+
if host == "non-native-windows":
|
|
462
|
+
raise GovernorError(f"a non-native Python ({sys.platform}) on Windows cannot create Job "
|
|
463
|
+
"Objects; run the governor with a native Windows python")
|
|
464
|
+
limits = load_limits(scope, budget, machine_budget)
|
|
465
|
+
env = dict(os.environ)
|
|
466
|
+
if host == "windows":
|
|
467
|
+
parent = verified_parent_percent(os.environ)
|
|
468
|
+
rate = effective_cpu_rate(limits, parent)
|
|
469
|
+
machine_pct = effective_machine_percent(limits, parent)
|
|
470
|
+
name = new_job_name()
|
|
471
|
+
job = create_job(limits, rate, name)
|
|
472
|
+
try:
|
|
473
|
+
assign_current_process(job)
|
|
474
|
+
except BaseException:
|
|
475
|
+
close_job(job) # safe: the assignment failed, so we are not a member
|
|
476
|
+
raise
|
|
477
|
+
reduce_to_query_handle(job) # the query handle stays open for the governor's lifetime
|
|
478
|
+
env["CADDIS_GOVERNED"] = scope
|
|
479
|
+
env["CADDIS_GOVERNED_JOB"] = name
|
|
480
|
+
env["CADDIS_GOVERNED_CPU_PERCENT"] = str(machine_pct)
|
|
481
|
+
sys.stderr.write(f"[caddis-governor] scope={scope} cpu={machine_pct}% "
|
|
482
|
+
f"mem={limits.memory_mb}MB procs={limits.max_processes} pid={os.getpid()}\n")
|
|
483
|
+
sys.stderr.flush()
|
|
484
|
+
return _run_child(argv, env)
|
|
485
|
+
wrapped = posix_argv(argv, limits)
|
|
486
|
+
if wrapped is None:
|
|
487
|
+
if not allow_uncapped:
|
|
488
|
+
raise GovernorError("no systemd-run on this platform, so no cap can be applied; "
|
|
489
|
+
"pass --allow-uncapped to run at lower priority only")
|
|
490
|
+
try:
|
|
491
|
+
os.nice(10)
|
|
492
|
+
how = "lower priority only"
|
|
493
|
+
except (AttributeError, OSError) as exc:
|
|
494
|
+
how = f"priority could NOT be lowered ({type(exc).__name__})"
|
|
495
|
+
sys.stderr.write(f"[caddis-governor] WARNING: --allow-uncapped: no CPU cap; {how}\n")
|
|
496
|
+
wrapped = argv # CADDIS_GOVERNED is NOT set: nothing downstream may believe a cap exists
|
|
497
|
+
else:
|
|
498
|
+
env["CADDIS_GOVERNED"] = scope
|
|
499
|
+
sys.stderr.flush()
|
|
500
|
+
return _run_child(wrapped, env)
|
|
501
|
+
except GovernorError as exc:
|
|
502
|
+
sys.stderr.write(f"[caddis-governor] REFUSING to run ungoverned: {exc}\n")
|
|
503
|
+
return EXIT_CONFIG
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def main(argv: list[str] | None = None) -> int:
|
|
507
|
+
raw = list(sys.argv[1:] if argv is None else argv)
|
|
508
|
+
command: list[str] = []
|
|
509
|
+
if "--" in raw:
|
|
510
|
+
i = raw.index("--")
|
|
511
|
+
raw, command = raw[:i], raw[i + 1:]
|
|
512
|
+
parser = argparse.ArgumentParser(prog="caddis_governor", description=__doc__.splitlines()[0])
|
|
513
|
+
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
514
|
+
p_exec = sub.add_parser("exec", help="run a command inside a capped job")
|
|
515
|
+
p_exec.add_argument("--scope", choices=SCOPES, required=True)
|
|
516
|
+
p_exec.add_argument("--budget", type=Path)
|
|
517
|
+
p_exec.add_argument("--machine-budget", type=Path, help=argparse.SUPPRESS) # additional ceiling only
|
|
518
|
+
p_exec.add_argument("--allow-uncapped", action="store_true",
|
|
519
|
+
help="where no cap can be applied (no systemd-run, macOS), run at lower priority anyway")
|
|
520
|
+
p_show = sub.add_parser("show", help="print the effective limits")
|
|
521
|
+
p_show.add_argument("--budget", type=Path)
|
|
522
|
+
p_show.add_argument("--machine-budget", type=Path, help=argparse.SUPPRESS)
|
|
523
|
+
args = parser.parse_args(raw)
|
|
524
|
+
|
|
525
|
+
if args.cmd == "show":
|
|
526
|
+
try:
|
|
527
|
+
for scope in SCOPES:
|
|
528
|
+
lim = load_limits(scope, args.budget, args.machine_budget)
|
|
529
|
+
print(f"{scope} cpu={lim.cpu_percent}% mem={lim.memory_mb}MB procs={lim.max_processes}")
|
|
530
|
+
except GovernorError as exc:
|
|
531
|
+
sys.stderr.write(f"[caddis-governor] {exc}\n")
|
|
532
|
+
return EXIT_CONFIG
|
|
533
|
+
return 0
|
|
534
|
+
|
|
535
|
+
if not command:
|
|
536
|
+
sys.stderr.write("usage: caddis_governor.py exec --scope session|lane [--budget PATH] -- <command...>\n")
|
|
537
|
+
return EXIT_USAGE
|
|
538
|
+
return exec_governed(command, args.scope, args.budget, args.machine_budget, args.allow_uncapped)
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
if __name__ == "__main__":
|
|
542
|
+
sys.exit(main())
|