@passioncode-ai/fabric-agent-adapter 0.76.0 → 0.78.0

package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_events.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 import re
+from types import MappingProxyType
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, List, Optional
 
@@ -33,6 +34,10 @@ EVENTS_MAX_LIMIT = 200
 LEVELS = ("info", "notice", "warning", "error")
 
 
+# The dashboard kit names its chip `warn`; an event written at that name is stored at the feed's `warning` (FAA-276).
+LEVEL_ALIASES = MappingProxyType({"warn": "warning"})
+
+
 _KIND = re.compile(r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*){0,5}\Z")
 
 
@@ -45,13 +50,15 @@ _SPAN_ID = re.compile(r"^(?!0{16}\Z)[0-9a-f]{16}\Z")
 def make_event(event_id: Any, at: str, kind: str, level: str, text: str, *, subject: Optional[Dict[str, str]] = None,
                link: Optional[str] = None, notify: bool = False, trace_id: Optional[str] = None,
                span_id: Optional[str] = None, data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
-    """One activity event: one sentence a person reads, never a machine id. `data` (DEC-0070) adds machine-readable
+    """One activity event: one sentence a person reads, never a machine id. `level` is one of `LEVELS`; `warn`, the
+    dashboard chip's name, is stored as `warning`. `data` (DEC-0070) adds machine-readable
     fields beside the sentence, `{schema, value}`, never a secret: it is refused here, so no feed carries one.
 
     An event about traced work carries its trace as a pair, `trace_id` and `span_id`
     (fabric-interop/0.1 C3.4 c); fabric_interop.trace_ids(traceparent) gives both."""
+    level = LEVEL_ALIASES.get(level, level) if isinstance(level, str) else level
     if level not in LEVELS:
-        raise ServiceError("level must be one of %s." % ", ".join(LEVELS))
+        raise ServiceError("level must be one of %s (or `warn` for `warning`)." % ", ".join(LEVELS))
     if not _KIND.match(kind):
         raise ServiceError("kind %r must be dotted lowercase." % kind)
     text = " ".join(str(text).split())

package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_peer_owner.py

@@ -1,6 +1,6 @@
 """peer_owner: which OS account owns each end of a loopback TCP connection.
 
-    peer_owner 1.1.0+kit.2: Observatory's peer_owner 1.1.0, vendored into the Fabric service kit as `fabric_peer_owner` (layer L0;
+    peer_owner 1.1.0+kit.3: Observatory's peer_owner 1.1.0, vendored into the Fabric service kit as `fabric_peer_owner` (layer L0;
     DEC-0154 HA-2, FAA-226) from Project Observatory's `observatory/engine/peer_owner.py` at
     commit f86f0622 (passioncode-ai/project-observatory-dashboard, the same licence). It differs
     from that file in this paragraph, in `Owner` and `Row` (NamedTuples here where it has
@@ -18,7 +18,8 @@
     Python without it reads netstat on macOS and imports the module everywhere. Since kit.2 (FAA-267), on
     macOS: the sysctl calibration also requires a child process's listener, an empty netstat reads as
     `TABLE_RESTRICTED`, and a caller may vouch for a listener with a pid it started (`pid_listens`,
-    `parse_lsof_listeners`, `LSOF`, `CheckedHTTPConnection(vouch_pid=...)`).
+    `parse_lsof_listeners`, `LSOF`, `CheckedHTTPConnection(vouch_pid=...)`, `own_connection(vouch_pid=...)`); since kit.3
+    the child's port is waited on with `selectors` and read under one deadline (FAA-273).
     A limit /proc sets: it carries a socket's creator and no pid, so a listening socket that root
     created and handed to another account (systemd socket activation with User=) reads as root's. Standard library only, no import of another kit module.
     The kit's tests are `test/test_fabric_peer_owner.py`, over `test/fixtures/peer-owner/`;
@@ -109,7 +110,7 @@ import time
 from typing import NamedTuple, Optional, Tuple
 
 # This file's own version, for projects that copy it; not the engine's (that is configuration.VERSION).
-PEER_OWNER_VERSION = "1.1.0+kit.2"  # Observatory's 1.1.0 with the kit's differences above
+PEER_OWNER_VERSION = "1.1.0+kit.3"  # Observatory's 1.1.0 with the kit's differences above
 
 THIS_ACCOUNT = "this-account"
 ANOTHER_ACCOUNT = "another-account"
@@ -500,10 +501,34 @@ _CHILD_LISTENER = ("import socket,sys\ns=socket.socket();s.bind(('127.0.0.1',0))
                    "print(s.getsockname()[1],flush=True);sys.stdin.read()")
 
 
+def _child_line(child: "subprocess.Popen[bytes]", deadline: float) -> Optional[bytes]:
+    """The child's first line, or None when no newline came by `deadline`. It waits with `selectors` (kqueue on
+    macOS: no FD_SETSIZE ceiling at fd 1024, which `select.select` refuses) and reads what is there, so a partial line
+    from an interpreter that then waits cannot hold the caller past the deadline (FAA-273 item 13, and the `selectors`
+    half of item 11)."""
+    import selectors  # POSIX: this reader runs on macOS alone
+    got = b""
+    try:
+        with selectors.DefaultSelector() as sel:
+            sel.register(child.stdout, selectors.EVENT_READ)
+            while b"\n" not in got and len(got) < 64:
+                left = deadline - time.monotonic()
+                if left <= 0 or not sel.select(left):
+                    return None
+                chunk = os.read(child.stdout.fileno(), 64)
+                if not chunk:
+                    return None
+                got += chunk
+    except (OSError, ValueError):
+        return None
+    return got.split(b"\n", 1)[0] if b"\n" in got else None
+
+
 def _child_listener_seen(rows_of) -> Optional[bool]:
     """Whether the table shows a listener of a child process of this one: False when it does not (the table shows
-    this process alone, FAA-267), None when no child could be started or did not answer within TIMEOUT_SECONDS (a
-    frozen or embedded host has no interpreter to start). The calibration trusts only True."""
+    this process alone, FAA-267), None when no child could be started (a frozen or embedded host has no interpreter to
+    start), or it gave no port line within TIMEOUT_SECONDS (silence, a partial or non-numeric line, an early exit, a
+    pipe that cannot be read). The calibration trusts only True."""
     if not sys.executable or getattr(sys, "frozen", False):
         return None
     try:
@@ -511,21 +536,25 @@ def _child_listener_seen(rows_of) -> Optional[bool]:
                                  stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
     except OSError:
         return None
+    deadline = time.monotonic() + TIMEOUT_SECONDS
+    port = None
     try:
-        import select  # POSIX: this reader runs on macOS alone
-        if not select.select([child.stdout], [], [], TIMEOUT_SECONDS)[0]:
-            return None
-        line = child.stdout.readline().strip()
-        if not line.isdigit():
+        line = _child_line(child, deadline)
+        if line is None or not line.strip().isdigit():
             return None
-        port = int(line)
-        return any(r.listening and r.local == ("127.0.0.1", port) and r.pid == child.pid for r in rows_of())
-    except (TableUnreadable, OSError, struct.error, ValueError):
-        return False
+        port = int(line.strip())
+        try:
+            return any(r.listening and r.local == ("127.0.0.1", port) and r.pid == child.pid for r in rows_of())
+        except (TableUnreadable, OSError, struct.error, ValueError):
+            return False
     finally:
+        # One deadline for the whole check: a child that gave no port is killed at once, and one that did gets what is
+        # left of the deadline to exit on its closed stdin before it is killed.
         try:
             child.stdin.close()
-            child.wait(timeout=TIMEOUT_SECONDS)
+            if port is None:
+                child.kill()
+            child.wait(timeout=max(0.0, deadline - time.monotonic()))
         except (OSError, subprocess.TimeoutExpired):
             child.kill()
             child.wait()

package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_service.py

@@ -275,6 +275,7 @@ from fabric_auth import (  # noqa: F401
 from fabric_events import (  # noqa: F401
     EVENTS_DEFAULT_LIMIT,
     EVENTS_MAX_LIMIT,
+    LEVEL_ALIASES,
     LEVELS,
     _KIND,
     _TRACE_ID,
@@ -313,6 +314,8 @@ from fabric_supervisors import (  # noqa: F401
     launchd_plist,
     _launchctl,
     _domain,
+    SYSTEM_DAEMONS,
+    launchd_domain,
     launchd_loaded,
     launchd_pid,
     fetch_well_known,

package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_supervisors.py

@@ -45,6 +45,10 @@ DEFAULT_EXIT_TIMEOUT = 15
 
 
 _LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,250}\Z")  # what launchd and the Node twin take as a label
+_USER_NAME = re.compile(r"^[a-z_][a-z0-9_.-]{0,31}\Z")  # a macOS short name, as a LaunchDaemon's UserName takes it
+
+#: Where a LaunchDaemon's plist lives: a job in launchd's system domain, started at boot with no GUI login (FAA-278).
+SYSTEM_DAEMONS = Path("/Library/LaunchDaemons")
 
 
 def _no_marker_argument(argv: Sequence[str]) -> None:
@@ -255,7 +259,7 @@ def task_uninstall(task: str, *, run: Callable[..., Any] = subprocess.run, syste
 
 def launchd_plist(label: str, program_arguments: Sequence[str], *, working_directory: Path,
                   stdout_path: Path, environment: Optional[Dict[str, str]] = None,
-                  exit_timeout: int = DEFAULT_EXIT_TIMEOUT) -> bytes:
+                  exit_timeout: int = DEFAULT_EXIT_TIMEOUT, user_name: Optional[str] = None) -> bytes:
     """RunAtLoad + KeepAlive true + ThrottleInterval 10; no secret may appear in `environment`.
 
     ``ExitTimeOut`` (15 s) sits above the drain and its hard exit (``Drain``: 8 + 2 s), so launchd's
@@ -264,7 +268,10 @@ def launchd_plist(label: str, program_arguments: Sequence[str], *, working_direc
 
     The same refusals as the Node twin (finding 10): a label launchd cannot name, no program, an ExitTimeOut that does
     not outlast the drain, an environment name ending TOKEN/SECRET/PASSWORD/KEY (a `*_FILE` path is fine), a value or
-    an argument shaped like a credential, and a control character in any string the plist carries."""
+    an argument shaped like a credential, and a control character in any string the plist carries.
+
+    ``user_name`` writes ``UserName``, the account a LaunchDaemon (system domain, FAA-278) runs as; it must be a macOS
+    short name, and it is meaningful only for a plist installed in ``/Library/LaunchDaemons``."""
     import fabric_secrets  # noqa: E402 - beside this file; the contract's detector (DEC-0044)
     if not _LABEL.match(str(label)):
         raise ServiceError("label must match %s" % _LABEL.pattern)
@@ -273,6 +280,8 @@ def launchd_plist(label: str, program_arguments: Sequence[str], *, working_direc
         raise ServiceError("A launchd job starts a program; ProgramArguments is empty.")
     if not exit_timeout > DRAIN_SECONDS:
         raise ServiceError("ExitTimeOut must outlast the %d s drain." % DRAIN_SECONDS)
+    if user_name is not None and not _USER_NAME.match(str(user_name)):
+        raise ServiceError("UserName must be a macOS short name matching %s" % _USER_NAME.pattern)
     env = dict(environment or {})
     for key, value in env.items():
         if re.search(r"(TOKEN|SECRET|PASSWORD|KEY)\Z", key) and not key.endswith("_FILE"):
@@ -303,6 +312,8 @@ def launchd_plist(label: str, program_arguments: Sequence[str], *, working_direc
         "StandardErrorPath": str(stdout_path),
         "EnvironmentVariables": env,
     }
+    if user_name is not None:
+        plist["UserName"] = str(user_name)
     return plistlib.dumps(plist)
 
 
@@ -317,16 +328,52 @@ def _domain() -> str:
     return "gui/%d" % os.getuid()
 
 
-def launchd_loaded(label: str) -> bool:
-    return _launchctl("print", "%s/%s" % (_domain(), label)).returncode == 0
+def launchd_domain(plist_path: Optional[Path] = None, domain: Optional[str] = None) -> str:
+    """The launchd domain a job lives in: ``system`` for a LaunchDaemon (``domain="system"``, or a plist in
+    ``/Library/LaunchDaemons``), else this user's ``gui/<uid>`` (FAA-278). A LaunchDaemon starts at boot, before any
+    login, so it is the form for a headless Mac that a FileVault restart (``fdesetup authrestart``) leaves with no GUI
+    session; a ``gui/<uid>`` job starts only while its user is logged in."""
+    if domain not in (None, "gui", "system"):
+        raise ServiceError("domain must be gui or system")
+    if domain is None and plist_path is not None:
+        folder = re.sub(r"^/{2,}", "/", str(Path(os.path.abspath(str(plist_path))).parent))  # POSIX keeps a leading //
+        domain = "system" if folder.lower() == str(SYSTEM_DAEMONS).lower() else "gui"  # APFS ignores case
+    if domain == "system":
+        if sys.platform != "darwin":
+            raise ServiceError("launchd is the supervisor on macOS only; use systemd_install on Linux and task_install on "
+                               "Windows.")
+        return "system"
+    return _domain()
+
+
+def _need_root(domain: str, label: str, plist_path: Path, install: bool) -> None:
+    """A system-domain job is installed and removed by root: refuse before anything is written (FAA-278)."""
+    if domain != "system" or os.geteuid() == 0:
+        return
+    if install:
+        raise ServiceError("A LaunchDaemon (launchd's system domain) is installed by root, and this process is not "
+                           "root: run launchd_install as root (the release, descriptor and token stay its UserName's, "
+                           "lifecycle reference, The system domain), or as root write %s and run "
+                           "`launchctl bootstrap system %s`. Status and pid read without root." % (plist_path, plist_path))
+    raise ServiceError("A LaunchDaemon (launchd's system domain) is removed by root, and this process is not root: run "
+                       "launchd_uninstall as root, or as root run `launchctl bootout system/%s`, remove %s and run "
+                       "`launchctl enable system/%s` (so a later install is not kept off). Status and pid read without "
+                       "root." % (label, plist_path, label))
+
+
+def launchd_loaded(label: str, domain: Optional[str] = None) -> bool:
+    """Whether launchd has the job ``label`` loaded in this user's ``gui/<uid>`` domain, or with ``domain="system"`` in
+    the system domain (FAA-278); reading needs no root."""
+    return _launchctl("print", "%s/%s" % (launchd_domain(domain=domain), label)).returncode == 0
 
 
 _PID_ROW = re.compile(r"^\s*pid\s*=\s*(\d+)\s*\Z")
 
 
-def launchd_pid(label: str) -> Optional[int]:
-    """The running pid of this account's launchd job ``label``, or None (not running, or not readable)."""
-    result = _launchctl("print", "%s/%s" % (_domain(), label))
+def launchd_pid(label: str, domain: Optional[str] = None) -> Optional[int]:
+    """The running pid of this account's launchd job ``label`` (with ``domain="system"``, of the LaunchDaemon), or
+    None (not running, or not readable). Reading needs no root."""
+    result = _launchctl("print", "%s/%s" % (launchd_domain(domain=domain), label))
     if result.returncode != 0:
         return None
     for line in (result.stdout or "").splitlines():
@@ -340,7 +387,8 @@ def _read_well_known(origin: str, timeout: float = 2.0,
                      vouch_pid: Optional[int] = None) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
     """(the well-known answer or None, why the listener was refused or None). A loopback origin is read only on a
     connection whose server end is this OS account's process (DEC-0154 HA-2): another account's, root's, or one whose
-    owner cannot be told is `foreign-user`, never this service's answer. `vouch_pid`, the pid of a launchd job this
+    owner cannot be told is never this service's answer, labelled `foreign-user`, or `table-restricted` when the socket
+    table hides other processes. `vouch_pid`, the pid of a launchd job this
     caller started, proves the listener when the socket table hides other processes (FAA-267)."""
     parts = urllib.parse.urlsplit(origin)
     if parts.scheme == "http" and parts.hostname == "127.0.0.1" and parts.port:
@@ -363,6 +411,107 @@ def _read_well_known(origin: str, timeout: float = 2.0,
         return None, None
 
 
+def _job_listens(pid: int, port: int) -> bool:
+    """Whether launchd's job `pid` listens on 127.0.0.1:<port> exactly (never the wildcard), asked of the process itself
+    (`lsof -p`), which a restricted socket table does not hide. Used only for a LaunchDaemon root just bootstrapped,
+    whose pid launchd itself named (FAA-278)."""
+    if sys.platform != "darwin" or not isinstance(pid, int) or isinstance(pid, bool) or pid <= 1:
+        return False
+    try:
+        out = _po._run([_po.LSOF, "-nP", "-a", "-p", str(pid), "-i4TCP", "-sTCP:LISTEN", "-Fn"])
+    except _po.TableUnreadable:
+        return False
+    return any(host == "127.0.0.1" and p == port for host, p in _po.parse_lsof_listeners(out))  # exact bind, as RT-3
+
+
+def _account_uid(user_name: str) -> Optional[int]:
+    """The uid of account ``user_name`` on this machine, or None when it has no such account. Only a short name: macOS
+    getpwnam also answers an upper-case name or the full name, which launchd's UserName does not take."""
+    if not isinstance(user_name, str) or not _USER_NAME.match(user_name):
+        return None
+    try:
+        import pwd
+        return pwd.getpwnam(user_name).pw_uid
+    except (ImportError, KeyError, ValueError):
+        return None
+
+
+def _pid_uid(pid: int) -> Optional[int]:
+    """The uid process ``pid`` runs as (``ps -o uid= -p``), or None when it is gone or cannot be read."""
+    try:
+        return _po.parse_ps_uid(_po._run([_po.PS, "-o", "uid=", "-p", str(pid)]))
+    except _po.TableUnreadable:
+        return None
+
+
+def _positive(value: Any) -> bool:
+    return isinstance(value, int) and not isinstance(value, bool) and value >= 0
+
+
+class _JobConnection(http.client.HTTPConnection):
+    """A connection whose server end must be launchd's job `job_pid`, running as `job_uid`, its UserName's uid (a
+    LaunchDaemon runs as its UserName, so the installer, root, reads it as another account's: the job's own pid and
+    uid are the proof, FAA-278; contract DEC-0161, proposed: RT-3 extended to launchd's system domain)."""
+
+    def __init__(self, host: str, port: int, job_pid: int, timeout: float, job_uid: Optional[int] = None) -> None:
+        if not isinstance(job_pid, int) or isinstance(job_pid, bool) or job_pid <= 1:
+            raise ServiceError("A LaunchDaemon's answer is read only on its job's pid, a positive number.")
+        if not _positive(job_uid):
+            raise ServiceError("A LaunchDaemon's answer is read only from its UserName's uid, a number.")
+        super().__init__(host, port, timeout=timeout)
+        self.job_pid = job_pid
+        self.job_uid = job_uid
+
+    def connect(self) -> None:
+        super().connect()
+        owner = _po.server_end(self.sock)
+        restricted = owner.kind == _po.UNKNOWN and (owner.reason or "").startswith(_po.TABLE_RESTRICTED)
+        if owner.pid == self.job_pid and not _job_listens(self.job_pid, self.port):
+            self.close()
+            # The job's pid, but lsof did not show it bound to exactly 127.0.0.1:<port>: a wildcard is never vouched for.
+            raise _po.ListenerRefused(_po.SERVER_NOT_OWN, "the LaunchDaemon's pid %d is not shown listening on exactly "
+                                      "127.0.0.1:%d by lsof (a wildcard is never vouched for); nothing was sent"
+                                      % (self.job_pid, self.port))
+        if owner.pid == self.job_pid or (restricted and _job_listens(self.job_pid, self.port)):
+            runs_as = _pid_uid(self.job_pid)
+            if runs_as == self.job_uid:
+                return
+            self.close()
+            # Not the restricted owner: the pid was proved, only its uid is wrong, and the error says so.
+            raise _po.ListenerRefused(_po.SERVER_NOT_OWN, "the LaunchDaemon's pid %d runs as uid %s, not its UserName's "
+                                      "uid %d; nothing was sent" % (self.job_pid, runs_as, self.job_uid))
+        self.close()
+        raise _po.ListenerRefused(_po.SERVER_NOT_OWN, "the server on port %d is not the LaunchDaemon's pid %d (%s): "
+                                  "another process holds the port, or the job's program starts the server as a child "
+                                  "(start it with exec); nothing was sent"
+                                  % (self.port, self.job_pid, owner.reason or owner.kind), owner)
+
+
+def _read_job_well_known(origin: str, job_pid: int, job_uid: Optional[int] = None,
+                         timeout: float = 2.0) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
+    """As `_read_well_known`, for a LaunchDaemon: a loopback origin is read only when launchd's job `job_pid`, running as
+    `job_uid`, accepted the connection; (the answer or None, why the listener was refused or None)."""
+    try:
+        port = urllib.parse.urlsplit(origin).port
+    except ValueError:  # a port out of range
+        port = None
+    if not (port and 1 <= port <= 65535 and origin == "http://127.0.0.1:%d" % port):
+        raise ServiceError("A LaunchDaemon's answer is read only at http://127.0.0.1:<port>, where its job's pid and uid "
+                           "prove it; %s is not." % origin)
+    conn = _JobConnection("127.0.0.1", port, job_pid, timeout, job_uid)
+    try:
+        conn.request("GET", "/.well-known/fabric-service", headers={"Host": "127.0.0.1:%d" % port})
+        response = conn.getresponse()
+        return (json.loads(response.read().decode()) if response.status == 200 else None), None
+    except _po.ListenerRefused as exc:
+        restricted = exc.owner is not None and (exc.owner.reason or "").startswith(_po.TABLE_RESTRICTED)
+        return None, "%s: %s" % (_po.TABLE_RESTRICTED if restricted else "foreign-user", exc)
+    except (OSError, ValueError, http.client.HTTPException):
+        return None, None
+    finally:
+        conn.close()
+
+
 def fetch_well_known(origin: str, timeout: float = 2.0) -> Optional[Dict[str, Any]]:
     """The well-known answer at `origin`, or None: nothing answers, the answer is not JSON, or the listener is not this
     OS account's process (DEC-0154 HA-2)."""
@@ -372,13 +521,13 @@ def fetch_well_known(origin: str, timeout: float = 2.0) -> Optional[Dict[str, An
 _OVERRIDE_ROW = re.compile(r'^\s*"([^"]+)"\s*=>\s*(\S+)\s*\Z')
 
 
-def launchd_override(label: str) -> Optional[str]:
+def launchd_override(label: str, domain: Optional[str] = None) -> Optional[str]:
     """The operator's launchd override for ``label``: ``"disabled"``, ``"enabled"``, None when
     none is recorded (a first install), or ``"unknown"`` when the table cannot be read.
 
-    Reads ``launchctl print-disabled gui/<uid>``; macOS 14+ prints ``=> disabled|enabled``,
+    Reads ``launchctl print-disabled gui/<uid>`` (``system`` with ``domain="system"``); macOS 14+ prints ``=> disabled|enabled``,
     older releases ``=> true|false`` (true means disabled)."""
-    result = _launchctl("print-disabled", _domain())
+    result = _launchctl("print-disabled", launchd_domain(domain=domain))
     if result.returncode != 0:
         return "unknown"
     for line in result.stdout.splitlines():
@@ -388,9 +537,63 @@ def launchd_override(label: str) -> Optional[str]:
     return None
 
 
+_PATH_ROW = re.compile(r"^\s*path\s*=\s*(.+?)\s*\Z")
+
+
+def _plist_field(path: Path, key: str) -> Optional[str]:
+    try:
+        value = plistlib.loads(Path(path).read_bytes()).get(key)
+    except Exception:  # unreadable or not a plist: it claims nothing
+        return None
+    return value if isinstance(value, str) else None
+
+
+def _same_file(a: Any, b: Any) -> bool:
+    """One file, whatever the case of its path (APFS ignores case, realpath keeps it)."""
+    try:
+        return os.path.samefile(str(a), str(b))
+    except OSError:
+        return os.path.realpath(str(a)) == os.path.realpath(str(b))
+
+
+def _label_holders(label: str, plist_path: Path) -> List[Path]:
+    """Other plists that hold launchd label ``label`` in the system domain (FAA-278): the plist launchd loaded the job
+    from, when it is not ``plist_path``, and any plist of that label beside ``plist_path``. Reading needs no root."""
+    found: List[Path] = []
+    result = _launchctl("print", "system/%s" % label)
+    if result.returncode == 0:
+        for line in (result.stdout or "").splitlines():
+            hit = _PATH_ROW.match(line)
+            if hit:
+                found.append(Path(hit.group(1)))
+                break
+    try:
+        siblings = sorted(Path(plist_path).parent.glob("*.plist"))
+    except OSError:
+        siblings = []
+    found += [p for p in siblings if _plist_field(p, "Label") == label]
+    return [p for p in dict.fromkeys(found) if not _same_file(p, plist_path)]
+
+
+def _answers(doc: Any, service_id: str, instance: str) -> bool:
+    """Whether a well-known answer names this service and instance; any other JSON is no answer."""
+    service = doc.get("service") if isinstance(doc, dict) else None
+    return isinstance(service, dict) and service.get("id") == service_id and service.get("instance") == instance
+
+
+def _refuse_held_label(label: str, plist_path: Path) -> None:
+    holders = _label_holders(label, plist_path)
+    if holders:
+        owner = _plist_field(holders[0], "UserName")
+        raise ServiceError("Label %s is already held by %s%s: the system domain is shared by every account of this "
+                           "Mac, so a label is the machine's and one plist holds it. Choose a label of your own (for "
+                           "example with the account in it), or remove that LaunchDaemon as root first."
+                           % (label, holders[0], " (%s's)" % owner if owner else ""))
+
+
 def launchd_install(label: str, plist_path: Path, plist_bytes: bytes, *, origin: str,
                     service_id: str, instance: str = "default", timeout: float = 40.0,
-                    force_enable: bool = False) -> Dict[str, Any]:
+                    force_enable: bool = False, domain: Optional[str] = None) -> Dict[str, Any]:
     """Write + lint the plist, bootout and WAIT for unload, bootstrap (retry EIO 5), then
     poll the well-known document until it answers with this identity.
 
@@ -398,38 +601,111 @@ def launchd_install(label: str, plist_path: Path, plist_bytes: bytes, *, origin:
     label. A label the operator disabled stays disabled and unloaded — the plist is still
     rewritten, so the new release is what starts once they turn it back on — and the answer is
     ``{"disabled": True, "label", "plist", "loaded"}`` instead of the well-known document.
-    ``force_enable`` is for an operator who asked for the service to be switched on."""
+    ``force_enable`` is for an operator who asked for the service to be switched on.
+
+    A plist in ``/Library/LaunchDaemons`` (or ``domain="system"``) is a LaunchDaemon in launchd's system domain, started
+    at boot with no login (FAA-278): only root installs it, refused before anything is written otherwise, and the
+    answer counts only when launchd's own job accepted the connection (its ``UserName``, not the installer, owns it).
+    A ``gui/<uid>`` bootstrap with no GUI session (SSH, a headless Mac after a FileVault restart) says so."""
     plist_path = Path(plist_path)
+    dom = launchd_domain(plist_path, domain)
+    which = "system" if dom == "system" else None
+    _need_root(dom, label, plist_path, True)
+    if dom == "system":
+        try:
+            port = urllib.parse.urlsplit(origin).port
+        except ValueError:  # a port out of range
+            port = None
+        if not (port and 1 <= port <= 65535 and origin == "http://127.0.0.1:%d" % port):
+            raise ServiceError("A LaunchDaemon's origin is http://127.0.0.1:<port>: only there is its answer proved by "
+                               "the job's pid and uid (FAA-278); %s is not." % origin)
+        try:
+            named = plistlib.loads(plist_bytes).get("UserName")
+        except Exception:  # a plist that does not parse fails plutil -lint below
+            named = None
+        named = named if isinstance(named, str) else None  # a UserName that is not a string names no account
+        if named:
+            _refuse_held_label(label, plist_path)
+        if named and plist_path.exists():
+            installed_label = _plist_field(plist_path, "Label")
+            if installed_label is not None and installed_label != label:
+                raise ServiceError("%s holds launchd label %s, not %s: writing over it would leave that job loaded from a "
+                                   "plist that no longer describes it. Uninstall it as root first, or write this one at "
+                                   "%s/%s.plist." % (plist_path, installed_label, label, SYSTEM_DAEMONS, label))
+            installed = _plist_field(plist_path, "UserName")
+            if installed and installed != named:
+                raise ServiceError("%s is %s's LaunchDaemon: the system domain is shared by every account of this Mac, so "
+                                   "a label is the machine's. Choose a label of your own (for example with the account in "
+                                   "it), or uninstall that one as root first." % (plist_path, installed))
+        if named and not _USER_NAME.match(named):
+            raise ServiceError("UserName %s is not a macOS short name (%s): launchd takes the account's short name."
+                               % (named, _USER_NAME.pattern))
+        job_uid = _account_uid(named) if named else None
+        if named and job_uid is None:
+            raise ServiceError("UserName %s is no account of this Mac: launchd would not start the job. Name the account "
+                               "whose Fabric and agents call the service." % named)
+        if job_uid == 0:
+            raise ServiceError("UserName %s is root: the job would run as root, keep its data in root's home, and every "
+                               "client refuses root's listener (DEC-0154 HA-2). Name the account whose Fabric and agents "
+                               "call the service." % named)
+        if not named:
+            raise ServiceError("A LaunchDaemon needs UserName: without it the job runs as root, keeps its data in root's "
+                               "home, and every client refuses root's listener (DEC-0154 HA-2). Write the plist with "
+                               "launchd_plist(..., user_name=<the account whose Fabric and agents call it>).")
     atomic_write(plist_path, plist_bytes, 0o644)
     lint = subprocess.run(["plutil", "-lint", str(plist_path)], capture_output=True, text=True)
     if lint.returncode != 0:
         raise ServiceError("plist failed plutil -lint: %s" % lint.stdout.strip())
-    target = "%s/%s" % (_domain(), label)
-    override = launchd_override(label)
+    target = "%s/%s" % (dom, label)
+    override = launchd_override(label, which)
     if force_enable or override is None:
         _launchctl("enable", target)
     elif override == "disabled":
-        return {"disabled": True, "label": label, "plist": str(plist_path), "loaded": launchd_loaded(label)}
+        return {"disabled": True, "label": label, "plist": str(plist_path), "loaded": launchd_loaded(label, which)}
     _launchctl("bootout", target)
     deadline = time.time() + 15
-    while launchd_loaded(label) and time.time() < deadline:
+    while launchd_loaded(label, which) and time.time() < deadline:
         time.sleep(0.25)
     last = ""
     for _ in range(5):
-        result = _launchctl("bootstrap", _domain(), str(plist_path))
+        result = _launchctl("bootstrap", dom, str(plist_path))
         if result.returncode == 0:
             break
         last = (result.stderr or result.stdout).strip()
         time.sleep(1.0)
     else:
+        if dom != "system" and re.search(r":\s*125:|Domain does not support|Could not find domain", last):
+            raise ServiceError("launchctl bootstrap failed: %s. This user has no GUI login session (an SSH session, or a "
+                               "headless Mac after a FileVault restart), so %s does not run jobs: remove the LaunchAgent "
+                               "this attempt wrote (launchd_uninstall), then install it as a LaunchDaemon in the system "
+                               "domain (a plist in %s with UserName, as root; FAA-278)." % (last, dom, SYSTEM_DAEMONS))
         raise ServiceError("launchctl bootstrap failed: %s" % last)
     deadline = time.time() + timeout
+    if dom == "system":
+        foreign = None
+        while time.time() < deadline:
+            job = launchd_pid(label, "system")
+            doc, why = _read_job_well_known(origin, job, job_uid) if job else (None, None)
+            foreign = why or foreign  # the last refusal stays the reason while launchd shows no pid
+            if _answers(doc, service_id, instance):
+                return doc
+            time.sleep(0.5)
+        loaded = ("; the job stays loaded and its plist starts it again at boot: as root, `launchctl bootout system/%s` "
+                  "and removing %s (or launchd_uninstall) take it away" % (label, plist_path))
+        if foreign and foreign.startswith(_po.TABLE_RESTRICTED):
+            raise ServiceError("%s could not be proved this LaunchDaemon's: %s; the job's pid did not show the listener: "
+                               "start the server with exec (no wrapper child), or read the job's log%s."
+                               % (origin, foreign, loaded))
+        if foreign:
+            raise ServiceError("%s is not this LaunchDaemon's: %s%s." % (origin, foreign, loaded))
+        raise ServiceError("%s did not answer as %s.%s within %.0f s; read its log%s."
+                           % (origin, service_id, instance, timeout, loaded))
     foreign = vouch = None
     while time.time() < deadline:
         doc, foreign = _read_well_known(origin, vouch_pid=vouch)
         if foreign and foreign.startswith(_po.TABLE_RESTRICTED):
             vouch = launchd_pid(label)  # only a restricted table needs the job's pid (FAA-267)
-        if doc and doc.get("service", {}).get("id") == service_id and doc.get("service", {}).get("instance") == instance:
+        if _answers(doc, service_id, instance):
             return doc
         time.sleep(0.5)
     if foreign:
@@ -444,20 +720,31 @@ def launchd_install(label: str, plist_path: Path, plist_bytes: bytes, *, origin:
 
 
 def launchd_uninstall(label: str, plist_path: Path, *, timeout: float = 30.0, purge: bool = False,
-                      service_id: Optional[str] = None, instance: str = "default") -> Dict[str, Any]:
+                      service_id: Optional[str] = None, instance: str = "default",
+                      domain: Optional[str] = None) -> Dict[str, Any]:
     """Symmetric with ``launchd_install`` (LC-14): bootout and WAIT until the job is gone, remove
     the plist, and reset the override to ``enabled`` — launchctl has no verb that deletes an
     override, and a leftover ``disabled`` would make a later fresh install stay off. Data stays
     unless ``purge`` (then ``service_id`` names whose data, logs and cache go, and ``instance`` which
     instance's: ``instance_dirs``, so one context's purge never reaches another's, FAA-125). A job still
-    loaded at ``timeout`` raises, leaving the plist and data in place."""
+    loaded at ``timeout`` raises, leaving the plist and data in place. A LaunchDaemon (a plist in
+    ``/Library/LaunchDaemons``, or ``domain="system"``) is removed only by root, and its data belongs to its UserName,
+    so ``purge`` is refused for it: remove the data as that user (FAA-278)."""
     if purge and not service_id:
         raise ServiceError("purge needs the service id whose data, logs and cache to remove.")
+    dom = launchd_domain(Path(plist_path), domain)
+    which = "system" if dom == "system" else None
+    _need_root(dom, label, Path(plist_path), False)
+    if purge and dom == "system":
+        raise ServiceError("A LaunchDaemon's data belongs to its UserName, not to root: uninstall without purge, then "
+                           "remove the data as that user.")
+    if dom == "system":
+        _refuse_held_label(label, Path(plist_path))  # a bootout would stop another plist's job (FAA-278)
     dirs = instance_dirs(str(service_id), instance) if purge else {}
-    target = "%s/%s" % (_domain(), label)
+    target = "%s/%s" % (dom, label)
     _launchctl("bootout", target)
     deadline = time.monotonic() + timeout
-    while launchd_loaded(label):
+    while launchd_loaded(label, which):
         if time.monotonic() >= deadline:
             raise ServiceError("launchd job %s is still loaded %.0f s after bootout; nothing was removed." % (label, timeout))
         time.sleep(0.1)