@passioncode-ai/fabric-agent-adapter 0.80.2 → 0.81.0

package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-supervisors.mjs

@@ -343,6 +343,8 @@ function guiDomain(platform, uid) {
 /** Where a LaunchDaemon's plist lives: a job in launchd's system domain, started at boot with no GUI login (FAA-278). */
 export const SYSTEM_DAEMONS = '/Library/LaunchDaemons';
 const USER_NAME = /^[a-z_][a-z0-9_.-]{0,31}$/;
+/** The short-name rule in words, printed alike by both twins (FAA-286 item 13). */
+export const USER_NAME_RULE = 'a lower-case letter or _, then up to 31 lower-case letters, digits, _, . or -';
 
 /**
  * The launchd domain a job lives in: `system` for a LaunchDaemon (`domain: 'system'`, or a plist in
@@ -351,6 +353,9 @@ const USER_NAME = /^[a-z_][a-z0-9_.-]{0,31}$/;
  */
 export function launchdDomain(plistPath = null, { domain = null, platform = process.platform, uid } = {}) {
   if (![null, undefined, 'gui', 'system'].includes(domain)) throw new ServiceError('domain must be gui or system');
+  if (plistPath !== null && plistPath !== undefined && /^(?:system|gui)(?:\/[0-9]*)?$/.test(String(plistPath))) {
+    throw new ServiceError(`${plistPath} is a launchd domain, not a plist path: pass { domain: '${String(plistPath) === 'system' ? 'system' : 'gui'}' } (FAA-286)`);
+  }
   let which = domain;
   if ((which === null || which === undefined) && plistPath !== null && plistPath !== undefined) {
     const paths = platform === 'darwin' ? path.posix : path; // a macOS path, whatever host reads it
@@ -363,6 +368,34 @@ export function launchdDomain(plistPath = null, { domain = null, platform = proc
   return guiDomain(platform, uid);
 }
 
+// A domain the plist's folder contradicts (FAA-286 items 14 and 21): a system job in a LaunchAgents folder (launchd also
+// loads it into gui/<uid> at login), or a gui job in /Library/LaunchDaemons (launchd loads it at boot as root). Another
+// folder is allowed for a system job, as the tests' and a staging install's are.
+function pairProblem(plistPath, domain, platform) {
+  const paths = platform === 'darwin' ? path.posix : path;
+  const folder = paths.dirname(paths.resolve(String(plistPath)));
+  const parts = String(plistPath).split(/[\\/]+/); // the folder's own name, whatever separator a test's host path uses
+  if (domain === 'system' && String(parts[parts.length - 2] ?? '').toLowerCase() === 'launchagents') {
+    return `A LaunchDaemon's plist lives in ${SYSTEM_DAEMONS}: ${plistPath} is in a LaunchAgents folder, which launchd also loads into gui/<uid> at login, so the job would run twice.`;
+  }
+  if (domain === 'gui' && folder.toLowerCase() === SYSTEM_DAEMONS.toLowerCase()) {
+    return `${plistPath} is in ${SYSTEM_DAEMONS}, which launchd loads at boot as a LaunchDaemon: a gui/<uid> job's plist goes in ~/Library/LaunchAgents.`;
+  }
+  return null;
+}
+
+// A LaunchAgent of `label` in `userName`'s ~/Library/LaunchAgents, which would run a second copy at its login (FAA-286
+// item 6); null when there is none or the home cannot be read.
+function leftoverAgent(label, userName, run) {
+  const read = run(['dscl', '.', '-read', `/Users/${userName}`, 'NFSHomeDirectory']);
+  const home = read?.status === 0 ? /NFSHomeDirectory:\s*(\S.*?)\s*$/m.exec(String(read.stdout ?? ''))?.[1] : null;
+  if (!home) return null;
+  const folder = path.join(home, 'Library', 'LaunchAgents');
+  let names = [];
+  try { names = fs.readdirSync(folder).filter((n) => n.endsWith('.plist')).sort(); } catch { return null; }
+  return names.map((n) => path.join(folder, n)).find((file) => topKey(plistXml(file, run), 'Label') === label) ?? null;
+}
+
 // A system-domain job is installed and removed by root: refuse before anything is written (FAA-278).
 function needRoot(domain, label, plistPath, install, euid) {
   if (domain !== 'system' || euid === 0) return;
@@ -390,6 +423,9 @@ function decodeXml(text) {
 function topStrings(text) {
   const token = /<!--[\s\S]*?-->|<\?[\s\S]*?\?>|<!DOCTYPE[^>]*>|<!\[CDATA\[([\s\S]*?)\]\]>|<(\/?)([A-Za-z][\w.:-]*)\s*[^>]*?(\/?)>|([^<]+)/g;
   const values = Object.create(null);
+  // The open containers: only the root dict's own keys are read (FAA-286 item 10: a root <array> holding a
+  // <key>UserName</key> is no dict, as plistlib reads it).
+  const open = [];
   let depth = 0;
   let key = null;
   let capture = null;
@@ -398,17 +434,21 @@ function topStrings(text) {
     for (let m = token.exec(String(text ?? '')); m; m = token.exec(String(text ?? ''))) {
       const [, cdata, closing, name, selfClosing, chars] = m;
       if (cdata !== undefined || chars !== undefined) {
-        if (capture) buffer += cdata !== undefined ? cdata : decodeXml(chars);
+        // every character reference, at any depth and in any value, is checked as expat checks it (FAA-286 item 10)
+        const decoded = chars !== undefined ? decodeXml(chars) : cdata;
+        if (capture) buffer += decoded;
       } else if (name === 'dict' || name === 'array') {
         if (selfClosing) {
           if (depth === 1 && key !== null) { delete values[key]; key = null; }
         } else if (closing) {
           depth -= 1;
+          open.pop();
         } else {
           if (depth === 1 && key !== null) { delete values[key]; key = null; } // a container is this key's value
           depth += 1;
+          open.push(name);
         }
-      } else if (name && depth === 1) {
+      } else if (name && depth === 1 && open[0] === 'dict') {
         if (selfClosing) {
           if (key !== null) {
             if (name === 'string') values[key] = '';
@@ -431,6 +471,7 @@ function topStrings(text) {
   } catch {
     return null;
   }
+  if (key !== null || capture !== null || depth !== 0) return null; // a dangling <key> or an open element: unreadable
   return values;
 }
 
@@ -569,8 +610,8 @@ function loopbackWellKnown(port, agent, host = '127.0.0.1') {
 
 // Whether launchd's job `pid` listens on exactly 127.0.0.1:<port>, never the wildcard, asked of the process itself (`lsof -p`),
 // which a restricted socket table does not hide. Only for a LaunchDaemon root bootstrapped, whose pid launchd named.
-async function jobListens(pid, port, run = runProgram) {
-  if (process.platform !== 'darwin' || !Number.isInteger(pid) || pid <= 1) return false;
+async function jobListens(pid, port, run = runProgram, platform = process.platform) {
+  if (platform !== 'darwin' || !Number.isInteger(pid) || pid <= 1) return false;
   try {
     const out = await run(LSOF, ['-nP', '-a', '-p', String(pid), '-i4TCP', '-sTCP:LISTEN', '-Fn']);
     return parseLsofListeners(out).some(([host, p]) => host === '127.0.0.1' && p === port); // exact bind, as RT-3
@@ -600,7 +641,11 @@ class JobAgent extends http.Agent {
     this.ask = ask;
   }
 
-  createConnection(options, callback) {
+  createConnection(options, given) {
+    // One answer per connection (FAA-286 item 12): an error during the async owner, lsof and ps checks would reach the
+    // callback a second time, as CheckedAgent's checkedSocket guards it.
+    let answered = false;
+    const callback = (err, socket) => { if (!answered) { answered = true; given(err, socket); } };
     const port = Number(options.port);
     const socket = net.connect({ host: '127.0.0.1', port, noDelay: true });
     const timer = setTimeout(() => socket.destroy(Object.assign(new Error('connect timeout'), { code: 'ETIMEDOUT' })), options.timeout ?? 10000);
@@ -614,14 +659,14 @@ class JobAgent extends http.Agent {
       } catch (err) {
         who = { kind: UNKNOWN, uid: null, pid: null, reason: `check: ${err?.name ?? 'Error'}` };
       }
-      if (who.pid === this.jobPid && !await jobListens(this.jobPid, port, this.ask.run)) {
+      if (who.pid === this.jobPid && !await jobListens(this.jobPid, port, this.ask.run, this.ask.platform)) {
         socket.destroy();
         // The job's pid, but lsof did not show it bound to exactly 127.0.0.1:<port>: a wildcard is never vouched for.
         callback(new ListenerRefused(SERVER_NOT_OWN, `the LaunchDaemon's pid ${this.jobPid} is not shown listening on exactly 127.0.0.1:${port} by lsof (a wildcard is never vouched for); nothing was sent`));
         return;
       }
       const proved = who.pid === this.jobPid || (who.kind === UNKNOWN && String(who.reason ?? '').startsWith(TABLE_RESTRICTED)
-        && await jobListens(this.jobPid, port, this.ask.run));
+        && await jobListens(this.jobPid, port, this.ask.run, this.ask.platform));
       if (!proved) {
         socket.destroy();
         callback(new ListenerRefused(SERVER_NOT_OWN, `the server on port ${port} is not the LaunchDaemon's pid ${this.jobPid} (${who.reason ?? who.kind}): another process holds the port, or the job's program starts the server as a child (start it with exec); nothing was sent`, who));
@@ -649,11 +694,11 @@ function loopbackPort(origin) {
 }
 
 // As wellKnownOf, for a LaunchDaemon: a loopback origin is read only when launchd's job `jobPid`, running as `jobUid`,
-// accepted the connection.
-export async function jobWellKnownOf(origin, { jobPid, jobUid } = {}) {
+// accepted the connection. `ask` reaches the owner, lsof and ps readers (a test's fakes; `platform` for lsof's).
+export async function jobWellKnownOf(origin, { jobPid, jobUid, ask = {} } = {}) {
   const port = loopbackPort(origin);
   if (port === null) throw new ServiceError(`A LaunchDaemon's answer is read only at http://127.0.0.1:<port>, where its job's pid and uid prove it; ${origin} is not.`);
-  return loopbackWellKnown(port, new JobAgent({ keepAlive: false, jobPid, jobUid }));
+  return loopbackWellKnown(port, new JobAgent({ keepAlive: false, jobPid, jobUid, ask }));
 }
 
 /**
@@ -672,9 +717,15 @@ export async function jobWellKnownOf(origin, { jobPid, jobUid } = {}) {
 export async function launchdInstall(label, plistPath, plistText, { origin, serviceId, instance = 'default', timeout = 40, forceEnable = false,
   run = defaultRun, platform = process.platform, uid, domain = null, euid = ownEuid(), fetchWellKnown = wellKnownOf,
   fetchJobWellKnown = jobWellKnownOf, sleep = pause } = {}) {
+  const pair = pairProblem(plistPath, domain, platform);
+  if (pair) throw new ServiceError(pair);
   const dom = launchdDomain(plistPath, { domain, platform, uid });
   launchdLabel(label);
   if (!origin || !serviceId) throw new ServiceError('launchdInstall needs the origin and the service id it waits to see answer.');
+  if (/^bplist/.test(Buffer.isBuffer(plistText) ? plistText.toString('latin1', 0, 6) : String(plistText ?? '').slice(0, 6))) {
+    // named as what it is, never as a missing UserName (FAA-286 item 7): the text launchdPlist writes is XML
+    throw new ServiceError('The plist is a binary plist: pass the XML text launchdPlist writes (plutil -convert xml1 converts one).');
+  }
   needRoot(dom, label, plistPath, true, euid);
   if (dom === 'system' && loopbackPort(origin) === null) {
     throw new ServiceError(`A LaunchDaemon's origin is http://127.0.0.1:<port>: only there is its answer proved by the job's pid and uid (FAA-278); ${origin} is not.`);
@@ -693,7 +744,7 @@ export async function launchdInstall(label, plistPath, plistText, { origin, serv
     }
   }
   if (dom === 'system' && named && !USER_NAME.test(named)) {
-    throw new ServiceError(`UserName ${named} is not a macOS short name (${USER_NAME.source}): launchd takes the account's short name.`);
+    throw new ServiceError(`UserName ${named} is not a macOS short name (${USER_NAME_RULE}): launchd takes the account's short name.`);
   }
   const jobUid = named ? accountUid(named, run) : null;
   if (dom === 'system' && named && jobUid === null) {
@@ -705,7 +756,12 @@ export async function launchdInstall(label, plistPath, plistText, { origin, serv
   if (dom === 'system' && !named) {
     throw new ServiceError("A LaunchDaemon needs UserName: without it the job runs as root, keeps its data in root's home, and every client refuses root's listener (DEC-0154 HA-2). Write the plist with launchdPlist(..., { userName: <the account whose Fabric and agents call it> }).");
   }
+  if (dom === 'system') {
+    const leftover = leftoverAgent(label, named, run);
+    if (leftover) throw new ServiceError(`${leftover} is a LaunchAgent of label ${label}: it would start a second copy at ${named}'s login beside this LaunchDaemon. Remove it first (launchdUninstall as ${named}; lifecycle reference, The system domain).`);
+  }
   const opts = { run, platform, uid, domain: dom === 'system' ? 'system' : null };
+  const existed = fs.existsSync(plistPath);
   atomicWrite(plistPath, plistText, 0o644);
   const lint = run(['plutil', '-lint', String(plistPath)]);
   if (lint?.status !== 0) throw new ServiceError(`plist failed plutil -lint: ${String(lint?.stdout || lint?.stderr || '').trim().slice(0, 200)}`);
@@ -725,6 +781,11 @@ export async function launchdInstall(label, plistPath, plistText, { origin, serv
       if (dom !== 'system' && /:\s*125:|Domain does not support|Could not find domain/.test(last)) {
         throw new ServiceError(`launchctl bootstrap failed: ${last.slice(0, 200)}. This user has no GUI login session (an SSH session, or a headless Mac after a FileVault restart), so ${dom} does not run jobs: remove the LaunchAgent this attempt wrote (launchdUninstall), then install it as a LaunchDaemon in the system domain (a plist in ${SYSTEM_DAEMONS} with UserName, as root; FAA-278).`);
       }
+      if (dom === 'system') {
+        // a plist left in /Library/LaunchDaemons loads at the next boot (FAA-286 item 15): one this call wrote goes
+        if (!existed) { try { fs.unlinkSync(plistPath); } catch { /* named below */ } }
+        throw new ServiceError(`launchctl bootstrap failed: ${last.slice(0, 200)}; ${fs.existsSync(plistPath) ? `${plistPath} stays and loads at the next boot: remove it as root` : 'the plist this call wrote was removed'}`);
+      }
       throw new ServiceError(`launchctl bootstrap failed: ${last.slice(0, 200)}`);
     }
     await sleep(1000);
@@ -780,6 +841,10 @@ export async function launchdUninstall(label, plistPath, { timeout = 30, purge =
   needRoot(dom, label, plistPath, false, euid);
   if (purge && dom === 'system') throw new ServiceError("A LaunchDaemon's data belongs to its UserName, not to root: uninstall without purge, then remove the data as that user.");
   if (dom === 'system') refuseHeldLabel(label, plistPath, run); // a bootout would stop another plist's job (FAA-278)
+  const holds = fs.existsSync(plistPath) ? topKey(plistXml(plistPath, run), 'Label') : null;
+  if (holds !== null && holds !== label) { // FAA-286 item 9: booting out one job and unlinking another's plist
+    throw new ServiceError(`${plistPath} holds launchd label ${holds}, not ${label}: nothing was removed. Name the plist of ${label}, or uninstall ${holds} with its own label.`);
+  }
   const dirs = purge ? instanceDirs(serviceId, instance, platform) : {}; // one instance's folders only (FAA-125)
   const opts = { run, platform, uid, domain: dom === 'system' ? 'system' : null };
   const target = `${dom}/${label}`;

package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_descriptor.py

@@ -5,12 +5,14 @@ from __future__ import annotations
 
 import hashlib
 import http.server
+import ipaddress
 import json
 import os
 import re
 import socket
 import socketserver
 import subprocess
+import threading
 import warnings
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Sequence, Tuple
@@ -88,19 +90,101 @@ _NOT_LAUNCHD = ("systemd", "task-scheduler", "node", "person")
 _AUTH_HEADER = re.compile(r"[A-Za-z][A-Za-z0-9-]{0,63}")
 
 
+#: How a loopback server treats the OS account of the process that connected (DEC-0154 HA-2's server side, HA-3;
+#: FAA-227): `refuse` (the default) closes a connection whose peer is another OS account's before reading a byte of
+#: it and serves the rest, labelled; `strict` also refuses a peer whose owner cannot be told and a system account
+#: other than root (uid 0, which can read every account's files and installs a LaunchDaemon); `off` reads no table.
+PEER_CHECKS = ("refuse", "strict", "off")
+
+
+def connection_refusal(owner: Any, mode: str = "refuse") -> Optional[str]:
+    """Why a server refuses the peer `owner` (a fabric_peer_owner.Owner) under `mode`, or None to serve it. The reason
+    is the owner's kind, or for `unknown` its reason; it names no address and nothing the peer sent."""
+    if mode not in PEER_CHECKS:
+        raise ValueError("peer_check must be one of %s" % ", ".join(PEER_CHECKS))
+    if mode == "off" or owner is None:
+        return None
+    kind = getattr(owner, "kind", fabric_peer_owner.UNKNOWN)
+    if kind == fabric_peer_owner.ANOTHER_ACCOUNT:
+        return kind
+    if mode == "strict" and kind != fabric_peer_owner.THIS_ACCOUNT:
+        if kind == fabric_peer_owner.SYSTEM and getattr(owner, "uid", None) == 0:
+            return None
+        return getattr(owner, "reason", None) or kind
+    return None
+
+
+def is_loopback_peer(client_address: Any) -> bool:
+    """Whether an accepted connection's peer address is a loopback one (127.0.0.0/8, ::1, or either IPv4-mapped): the
+    only peers whose owner can be another account of this computer (DEC-0154 HA-2). An address that cannot be read
+    counts as loopback, so its owner is still read from the table."""
+    try:
+        return ipaddress.ip_address(fabric_peer_owner.norm_host(str(client_address[0]))).is_loopback
+    except Exception:
+        return True
+
+
 class LoopbackHTTPServer(http.server.ThreadingHTTPServer):
     """ThreadingHTTPServer for a loopback service: threads are daemons, and the bind asks
     no resolver. HTTPServer.server_bind() calls socket.getfqdn() between bind() and listen();
     on a Mac with a slow resolver the port then stays bound but silent, so a host sees
-    neither an answer nor a refusal."""
+    neither an answer nor a refusal.
+
+    Each accepted connection's peer is told from the computer's socket table (`fabric_peer_owner.peer`) in the
+    connection's own thread, before the handler reads a byte (DEC-0154 HA-2's server side, FAA-227). `peer_check`
+    (`PEER_CHECKS`) decides which owners are closed unread; a served handler finds its peer's owner with
+    `self.server.peer_owner()`. A refusal writes one line to stderr with the owner's kind, uid and pid and nothing the
+    peer sent (`peer_refused`), and counts in `peer_refusals`. The check applies to loopback peers only: a peer at
+    another address (a service bound to 0.0.0.0) is no local account, so its table is not read and it is served under
+    `refuse` and `strict` alike, with `peer_owner()` None."""
 
     daemon_threads = True
+    peer_check = "refuse"
+    #: The reader of an accepted socket's peer; a test replaces it.
+    peer_reader = staticmethod(lambda sock: fabric_peer_owner.peer(sock))
+
+    def __init__(self, server_address: Any, handler: Any, bind_and_activate: bool = True, *,
+                 peer_check: Optional[str] = None) -> None:
+        mode = self.peer_check if peer_check is None else peer_check
+        connection_refusal(None, mode)  # refuses an unknown mode before the port is bound
+        self.peer_check = mode
+        self.peer_refusals = 0
+        self._peer = threading.local()
+        super().__init__(server_address, handler, bind_and_activate)
 
     def server_bind(self) -> None:
         socketserver.TCPServer.server_bind(self)
         host, port = self.server_address[:2]
         self.server_name, self.server_port = host, port
 
+    def peer_owner(self) -> Any:
+        """The owner of the peer this thread's handler serves (None under `peer_check="off"`)."""
+        return getattr(self._peer, "owner", None)
+
+    def finish_request(self, request: Any, client_address: Any) -> None:
+        owner = None
+        if self.peer_check != "off" and is_loopback_peer(client_address):
+            try:
+                owner = self.peer_reader(request)
+            except Exception as exc:  # a reader that fails is an owner nobody can tell, never a crash of the server
+                owner = fabric_peer_owner.Owner(fabric_peer_owner.UNKNOWN, None, None, "none",
+                                                "reader: %s" % type(exc).__name__)
+        reason = connection_refusal(owner, self.peer_check)
+        if reason is not None:
+            self.peer_refusals += 1
+            self.peer_refused(owner, reason)
+            return  # process_request_thread closes the socket unread (shutdown_request)
+        self._peer.owner = owner
+        try:
+            super().finish_request(request, client_address)
+        finally:
+            self._peer.owner = None
+
+    def peer_refused(self, owner: Any, reason: str) -> None:
+        """One line on stderr per refused connection; override to log it elsewhere. Never the peer's bytes."""
+        _sys.stderr.write("fabric: refused a loopback peer (%s, uid %s, pid %s) before reading it (DEC-0154)\n"
+                          % (reason, getattr(owner, "uid", None), getattr(owner, "pid", None)))
+
 
 def descriptor_path(service_id: str, instance: str = "default", directory: Optional[Path] = None) -> Path:
     return (directory or services_dir()) / ("%s.%s.json" % (service_id, instance))

package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_holds.py

@@ -0,0 +1,316 @@
+"""Work a service holds by its own limits, said to its caller and its operator (FAA-298). Part of the service kit
+(layer L3, after fabric_interop).
+
+A service that holds work by a limit of its own (a daily quota, a per-caller share, a concurrency cap, a money cap)
+says so three ways:
+
+- to the caller, on the job: `waiting_message(reason, since, until, message)` is the job's `statusMessage`. The pinned
+  `interop-job` schema is closed (`additionalProperties: false`), so `waiting {reason, since, until, message}` cannot
+  be a field of the job; it rides the contract's pause grammar (DEC-0087) as its text:
+  `quota-exhausted[ until <until>]: waiting <reason> since <since>[: <message>]`. A caller that reads only the pause
+  grammar waits (and until `until`, when given); `read_waiting(answer)` gives the four parts back. `JobStore.pause`
+  takes the same statusMessage through `hold_job`.
+- to the operator, once per hold: `HoldBook.hold` opens a hold per (reason, caller), or joins the open one, and returns
+  the push only when it opens: one `notify` event naming the caller, the work, the reason and when the hold frees by
+  itself, with the question whether to raise the limit. Further jobs held by the same limit for the same caller join
+  the hold silently; a hold is closed by `release`, by its `until` passing, or by a lift that covers it.
+- the answer, as a scoped lift: `HoldBook.lift` records the operator's words (a DEC-0121 decision, action
+  `limit.lift`) with its scope (the reason, optionally one caller and one job) and its size (a number of jobs, an
+  amount of money, or a raised limit for a time window such as 100 a day for today). `admit` spends a jobs or money
+  lift; `limit` gives the limit in force. A lift falls back by itself when its window ends or its amount is spent.
+
+`HoldBook` keeps one JSON file (0600, written atomically); a ThreadingHTTPServer's handlers share it under the store
+lock. It holds caller names, work labels, reasons and the operator's words, never a credential: `hold` refuses a text
+the contract's detector reads as one.
+"""
+from __future__ import annotations
+
+import json
+import re
+import uuid
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+import sys as _sys  # noqa: E402
+from pathlib import Path as _Path  # noqa: E402
+if str(_Path(__file__).resolve().parent) not in _sys.path:  # the kit loads from any working directory (FAA-65)
+    _sys.path.insert(0, str(_Path(__file__).resolve().parent))
+from fabric_base import ServiceError, now_iso, store_lock  # noqa: E402
+from fabric_paths import atomic_write  # noqa: E402
+from fabric_actions import decision_shape  # noqa: E402
+from fabric_interop_rules import pause_message, read_job_pause, utc_time, QUOTA_EXHAUSTED  # noqa: E402
+import fabric_secrets  # noqa: E402
+
+#: The limits a hold names: a daily quota, a per-caller share, a concurrency cap, a money cap.
+HOLD_REASONS = ("daily-quota", "caller-share", "concurrency", "money")
+#: How the push names each reason to a person.
+REASON_WORDS = {"daily-quota": "the daily quota", "caller-share": "its per-caller share",
+                "concurrency": "the concurrency cap", "money": "the money cap"}
+#: The push's kind: a decision for the operator (level notice, notify true; events-and-notifications.md).
+HOLD_EVENT_KIND = "limit.held"
+#: The record of a lift applied (level info, no notify).
+LIFT_EVENT_KIND = "limit.lifted"
+#: The decision action a lift carries (DEC-0121 AP-2 shape).
+LIFT_ACTION = "limit.lift"
+_MAX_STATUS_MESSAGE = 500
+_WAITING = re.compile(r"^waiting (daily-quota|caller-share|concurrency|money) since ([0-9][0-9TZ:.-]*?)(?:: ([\s\S]+))?\Z")
+_SIZES = ("jobs", "usd", "limit")
+
+
+def _instant(value: Any, name: str) -> str:
+    if utc_time(value) is None:
+        raise ServiceError("%s is a UTC instant (Z, at most 3 fraction digits)." % name)
+    return value
+
+
+def waiting_message(reason: str, since: str, until: Optional[str] = None, message: Optional[str] = None) -> str:
+    """A held job's statusMessage: the pause grammar, its text `waiting <reason> since <since>[: <message>]`. The
+    message is shortened, never refused, to fit the contract's 500 characters; a reason or an instant outside the
+    grammar raises ServiceError."""
+    if reason not in HOLD_REASONS:
+        raise ServiceError("A hold's reason is one of %s." % ", ".join(HOLD_REASONS))
+    _instant(since, "since")
+    if until is not None:
+        _instant(until, "until")
+    text = "waiting %s since %s" % (reason, since)
+    head = len(QUOTA_EXHAUSTED) + (0 if until is None else len(" until ") + len(until)) + len(": ")
+    if message is not None:
+        message = " ".join(str(message).split())
+        room = _MAX_STATUS_MESSAGE - head - len(text) - len(": ")
+        if message and room > 0:
+            text += ": " + (message if len(message) <= room else message[:room - 3].rstrip() + "...")
+    return pause_message(until, text)
+
+
+def read_waiting(answer: Any) -> Optional[Dict[str, Any]]:
+    """`{reason, since, until?, message?}` of a fabric.job.get answer whose job a limit holds (`waiting_message`), else
+    None: a plain pause, a queue position or a finished job is not a hold."""
+    pause = read_job_pause(answer)
+    if not pause.get("paused"):
+        return None
+    status = answer["job"]["statusMessage"]
+    rest = status[len(QUOTA_EXHAUSTED):]
+    if "resetsAt" in pause:
+        rest = rest[len(" until ") + len(pause["resetsAt"]):]
+    if not rest.startswith(": "):
+        return None
+    match = _WAITING.match(rest[2:])
+    if not match or utc_time(match.group(2)) is None:
+        return None
+    waiting: Dict[str, Any] = {"reason": match.group(1), "since": match.group(2)}
+    if "resetsAt" in pause:
+        waiting["until"] = pause["resetsAt"]
+    if match.group(3) is not None:
+        waiting["message"] = match.group(3)
+    return waiting
+
+
+def hold_job(jobs: Any, job_id: str, reason: str, since: str, until: Optional[str] = None,
+             message: Optional[str] = None) -> Dict[str, Any]:
+    """Put one job of a fabric_interop JobStore on hold: working, its statusMessage `waiting_message(...)`, its
+    pollIntervalMs reaching `until` (JobStore.pause). Returns the job."""
+    text = waiting_message(reason, since, until, message)[len(QUOTA_EXHAUSTED):]
+    if until is not None:
+        text = text[len(" until ") + len(until):]
+    return jobs.pause(job_id, resets_at=until, text=text[2:])
+
+
+def hold_sentence(caller: str, work: str, reason: str, until: Optional[str]) -> str:
+    """The push's one sentence: who, what, why, when it frees by itself, and the question."""
+    frees = ("it frees by itself at %s" % until) if until is not None else (
+        "it frees by itself when a running job ends" if reason in ("concurrency", "caller-share")
+        else "it does not free by itself")
+    return "%s for %s is held by %s; %s. Raise the limit for it?" % (work, caller, REASON_WORDS[reason], frees)
+
+
+def _text(value: Any, name: str, high: int = 200) -> str:
+    if not isinstance(value, str) or not value.strip() or len(value) > high:
+        raise ServiceError("%s is a text of 1 to %d characters." % (name, high))
+    if fabric_secrets.secret_value(value):
+        raise ServiceError("%s looks like a credential; a hold names callers and work, never a secret." % name)
+    return " ".join(value.split())
+
+
+class HoldBook:
+    """The holds a service's limits put on work and the operator's lifts, in one file (FAA-298).
+
+    `events`, when given, is the service's event log (an object with `append(kind, level, text, subject=..., **extra)`,
+    such as fabric_events.JsonlEventLog): the push and the lift record are written there. Without it they are returned
+    for the service to write."""
+
+    def __init__(self, path: Path, events: Any = None, clock: Callable[[], str] = now_iso):
+        self.path = Path(path)
+        self.events = events
+        self.clock = clock
+        self._lock = store_lock(self.path)
+
+    # --- the file -------------------------------------------------------------------------
+    def _load(self) -> Dict[str, Any]:
+        if not self.path.exists():
+            return {"holds": [], "lifts": []}
+        try:
+            data = json.loads(self.path.read_text(encoding="utf-8"))
+        except ValueError:
+            raise ServiceError("The hold book %s is not JSON; it is kept as it is." % self.path.name) from None
+        if not isinstance(data, dict) or not isinstance(data.get("holds"), list) or not isinstance(data.get("lifts"), list):
+            raise ServiceError("The hold book %s is not {holds, lifts}; it is kept as it is." % self.path.name)
+        return data
+
+    def _save(self, data: Dict[str, Any]) -> None:
+        atomic_write(self.path, (json.dumps(data, ensure_ascii=False, indent=1) + "\n").encode(), 0o600)
+
+    def _now(self, now: Optional[str]) -> str:
+        return _instant(now if now is not None else self.clock(), "now")
+
+    def _emit(self, kind: str, level: str, text: str, subject: Dict[str, str], notify: bool) -> Dict[str, Any]:
+        if self.events is not None:
+            return self.events.append(kind, level, text, subject=subject, notify=notify)
+        out: Dict[str, Any] = {"kind": kind, "level": level, "text": text, "subject": subject}
+        if notify:
+            out["notify"] = True
+        return out
+
+    @staticmethod
+    def _expire(data: Dict[str, Any], at: int) -> None:
+        for hold in data["holds"]:
+            if hold.get("closed") is None and hold.get("until") is not None and utc_time(hold["until"]) <= at:
+                hold["closed"], hold["closedAt"] = "expired", hold["until"]
+
+    @staticmethod
+    def _live(lift: Dict[str, Any], at: int) -> bool:
+        if lift.get("from") is not None and utc_time(lift["from"]) > at:
+            return False
+        if lift.get("until") is not None and utc_time(lift["until"]) <= at:
+            return False
+        return lift.get("left", 1) > 0
+
+    # --- holds ----------------------------------------------------------------------------
+    def hold(self, *, caller: str, work: str, reason: str, until: Optional[str] = None, message: Optional[str] = None,
+             job_id: Optional[str] = None, now: Optional[str] = None) -> Dict[str, Any]:
+        """Hold `work` for `caller` by `reason`. Returns {hold, statusMessage, push}: `push` is the notify event when this
+        call opened the hold, None when the work joined an open one (one push per hold)."""
+        caller, work = _text(caller, "caller"), _text(work, "work")
+        if reason not in HOLD_REASONS:
+            raise ServiceError("A hold's reason is one of %s." % ", ".join(HOLD_REASONS))
+        at = self._now(now)
+        if until is not None and utc_time(_instant(until, "until")) <= utc_time(at):
+            raise ServiceError("until %s has passed: the limit no longer holds the work." % until)
+        with self._lock:
+            data = self._load()
+            self._expire(data, utc_time(at))
+            hold = next((h for h in data["holds"] if h.get("closed") is None and h["reason"] == reason
+                         and h["caller"] == caller), None)
+            opened = hold is None
+            if opened:
+                hold = {"id": uuid.uuid4().hex, "reason": reason, "caller": caller, "work": work, "since": at,
+                        "until": until, "jobs": [], "closed": None}
+                data["holds"].append(hold)
+            if job_id is not None and job_id not in hold["jobs"]:
+                hold["jobs"].append(str(job_id))
+            self._save(data)
+        status = waiting_message(reason, hold["since"], hold["until"], message)
+        push = None
+        if opened:
+            push = self._emit(HOLD_EVENT_KIND, "notice", hold_sentence(caller, work, reason, until),
+                              {"type": "limit", "id": "%s:%s" % (reason, caller)[:200], "label": REASON_WORDS[reason]}, True)
+        return {"hold": dict(hold), "statusMessage": status, "push": push}
+
+    def release(self, *, reason: str, caller: str, now: Optional[str] = None) -> Optional[Dict[str, Any]]:
+        """Close the open hold of (reason, caller): the limit freed the work. The next hold pushes again."""
+        at = self._now(now)
+        with self._lock:
+            data = self._load()
+            hold = next((h for h in data["holds"] if h.get("closed") is None and h["reason"] == reason
+                         and h["caller"] == caller), None)
+            if hold is None:
+                return None
+            hold["closed"], hold["closedAt"] = "released", at
+            self._save(data)
+            return dict(hold)
+
+    def open_holds(self, now: Optional[str] = None) -> List[Dict[str, Any]]:
+        data = self._load()
+        self._expire(data, utc_time(self._now(now)))
+        return [dict(h) for h in data["holds"] if h.get("closed") is None]
+
+    # --- lifts ----------------------------------------------------------------------------
+    def lift(self, *, reason: str, decision: Dict[str, Any], caller: Optional[str] = None, job_id: Optional[str] = None,
+             jobs: Optional[int] = None, usd: Optional[float] = None, limit: Optional[int] = None,
+             until: Optional[str] = None, now: Optional[str] = None) -> Dict[str, Any]:
+        """Record the operator's answer as a scoped lift: exactly one of `jobs` (that many more jobs), `usd` (that much
+        more money) or `limit` (the limit raised to that value, which needs `until`, the window's end). `decision` is
+        the operator's words in decision.schema.json's shape with action `limit.lift`. The open hold it covers closes
+        as lifted. Returns the lift."""
+        if reason not in HOLD_REASONS:
+            raise ServiceError("A lift's reason is one of %s." % ", ".join(HOLD_REASONS))
+        if not decision_shape(decision) or decision.get("action") != LIFT_ACTION:
+            raise ServiceError("A lift carries the operator's decision (decision.schema.json, action limit.lift, their "
+                               "words verbatim): fabric_actions.operator_decision('limit.lift', words).")
+        sizes = {"jobs": jobs, "usd": usd, "limit": limit}
+        given = [name for name in _SIZES if sizes[name] is not None]
+        if len(given) != 1:
+            raise ServiceError("A lift is one of jobs, usd or limit.")
+        size = sizes[given[0]]
+        if isinstance(size, bool) or not isinstance(size, (int, float)) or size <= 0 or (
+                given[0] != "usd" and int(size) != size):
+            raise ServiceError("A lift's %s is a positive %s." % (given[0], "number" if given[0] == "usd" else "integer"))
+        at = self._now(now)
+        if until is not None and utc_time(_instant(until, "until")) <= utc_time(at):
+            raise ServiceError("A lift's window ends after now.")
+        if given[0] == "limit" and until is None:
+            raise ServiceError("A raised limit needs its window's end (until): it falls back when the window ends.")
+        record: Dict[str, Any] = {"id": uuid.uuid4().hex, "reason": reason, "caller": None if caller is None else _text(caller, "caller"),
+                                  "job": None if job_id is None else str(job_id), "from": at, "until": until,
+                                  given[0]: size, "decision": dict(decision), "at": at}
+        if given[0] != "limit":
+            record["left"] = size
+        with self._lock:
+            data = self._load()
+            self._expire(data, utc_time(at))
+            for hold in data["holds"]:
+                if hold.get("closed") is None and hold["reason"] == reason and record["caller"] in (None, hold["caller"]) and (
+                        job_id is None or str(job_id) in hold["jobs"]):
+                    hold["closed"], hold["closedAt"], hold["lift"] = "lifted", at, record["id"]
+            data["lifts"].append(record)
+            self._save(data)
+        scope = ("for %s" % record["caller"]) if record["caller"] else "for every caller"
+        amount = {"jobs": "%s more job(s)", "usd": "USD %s more", "limit": "the limit at %s"}[given[0]] % size
+        self._emit(LIFT_EVENT_KIND, "info", "The operator lifted %s %s: %s%s." % (
+            REASON_WORDS[reason], scope, amount, (" until %s" % until) if until else ""),
+            {"type": "limit", "id": "%s:%s" % (reason, record["caller"] or "*"), "label": REASON_WORDS[reason]}, False)
+        return dict(record)
+
+    def _matching(self, data: Dict[str, Any], reason: str, caller: Optional[str], job_id: Optional[str], at: int,
+                  kind: str) -> List[Dict[str, Any]]:
+        return [l for l in data["lifts"] if l["reason"] == reason and kind in l and self._live(l, at)
+                and l.get("caller") in (None, caller) and l.get("job") in (None, job_id)]
+
+    def admit(self, *, reason: str, caller: str, job_id: Optional[str] = None, cost_usd: Optional[float] = None,
+              now: Optional[str] = None) -> Dict[str, Any]:
+        """Whether a lift lets work past the limit that would hold it: {admitted: True, lift} spends one job of a jobs
+        lift, or `cost_usd` of a money lift (the narrowest scope first: a job's, a caller's, every caller's), else
+        {admitted: False}. A raised limit is read with `limit`, not spent here."""
+        at = utc_time(self._now(now))
+        with self._lock:
+            data = self._load()
+            kind = "usd" if cost_usd is not None else "jobs"
+            for lift in sorted(self._matching(data, reason, caller, None if job_id is None else str(job_id), at, kind),
+                               key=lambda l: (l.get("job") is None, l.get("caller") is None)):
+                need = cost_usd if kind == "usd" else 1
+                if lift["left"] >= need:
+                    lift["left"] = round(lift["left"] - need, 6) if kind == "usd" else lift["left"] - 1
+                    self._save(data)
+                    return {"admitted": True, "lift": lift["id"], "left": lift["left"]}
+            return {"admitted": False}
+
+    def limit(self, *, reason: str, base: float, caller: Optional[str] = None, now: Optional[str] = None) -> float:
+        """The limit in force: `base`, or the highest live raised limit for (reason, caller); `base` again once the
+        window ends."""
+        at = utc_time(self._now(now))
+        raised = [l["limit"] for l in self._matching(self._load(), reason, caller, None, at, "limit")]
+        return max([base] + raised)
+
+    def lifts(self, now: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Every lift with `live`: whether it still applies now."""
+        at = utc_time(self._now(now))
+        return [dict(l, live=self._live(l, at)) for l in self._load()["lifts"]]