@scp-dev/broker 0.4.2 → 0.5.0

package/dist/index.d.ts

@@ -234,6 +234,8 @@ interface Bridge {
     }): Promise<SendResult>;
     /** a local endpoint was claimed, let go or changed its status: mirrored as a delta */
     changed(endpoint: string): void;
+    /** mirrors these endpoints now and resolves true once the hub took the delta, false when it did not or none is mirrored */
+    mirrorNow?(endpoints: string[]): Promise<boolean>;
     /** rules, caps, groups or a title changed, which can touch several endpoints: every mirrored endpoint is sent again */
     resync(): void;
     /** a local endpoint took another handle: the hub renames its mirror in place, keeping what queued for it */
@@ -244,6 +246,13 @@ interface Bridge {
     local?(event: LocalDelivery): void;
     /** the endpoints of other accounts this account's contacts let it reach */
     contacts(): Promise<XListed[]>;
+    /** sets (or with a null url clears) the URL the hub calls to wake this machine; the secret signs those calls and is never shown again */
+    machineWake?(params: {
+        url: string | null;
+        secret?: string;
+    }): Promise<{
+        set: boolean;
+    }>;
     status(): BridgeStatus;
     /** reads the credential again and reconnects now, also after the hub revoked or replaced this bridge */
     relink(): void;
@@ -340,6 +349,10 @@ interface BrokerOptions {
     env?: Record<string, string | undefined> | undefined;
     /** the judge model client for a policy's `judge.model` (default the provider's HTTP API) */
     judgeClient?: JudgeClientFactory | undefined;
+    /** the wake hooks' configuration file (default wakers.json beside the policy file), read again when it changes */
+    wakersPath?: string | undefined;
+    /** the wake hooks by provider, in place of the file */
+    wakers?: Record<string, unknown> | undefined;
 }
 /** what a link gets from the broker: its core, the remote hooks a bridge gets, and a send as a local endpoint or a mirrored one through the judge */
 interface LinkContext {

package/dist/index.js

@@ -446,16 +446,16 @@ var RelayClient = class _RelayClient {
         this.fail(new Error("the relay broker sent a line that is not JSON"));
         return;
       }
-      const message = parsed;
-      if (message.method) {
-        for (const fn of this.listeners) fn(message.method, message.params);
+      const message2 = parsed;
+      if (message2.method) {
+        for (const fn of this.listeners) fn(message2.method, message2.params);
         continue;
       }
-      const pending = message.id !== void 0 ? this.pending.get(message.id) : void 0;
+      const pending = message2.id !== void 0 ? this.pending.get(message2.id) : void 0;
       if (!pending) continue;
-      this.pending.delete(message.id);
-      if (message.error) pending.reject(new RelayError(message.error.data?.code ?? "invalid_params", message.error.message, message.error.data?.retry_after));
-      else pending.resolve(message.result);
+      this.pending.delete(message2.id);
+      if (message2.error) pending.reject(new RelayError(message2.error.data?.code ?? "invalid_params", message2.error.message, message2.error.data?.retry_after));
+      else pending.resolve(message2.result);
     }
   }
   fail(error) {
@@ -534,6 +534,7 @@ import {
   XLIMITS
 } from "@scp-dev/core";
 import { HUB_LIMITS, LOCAL_EVENTS } from "@scp-dev/hub";
+var MIRROR_NOW_BATCH = 32;
 var LOCAL_BUFFER = 1e3;
 var idHash = (id) => createHash2("sha256").update(id).digest("hex").slice(0, 32);
 var CONNECT_TIMEOUT_MS = 15e3;
@@ -822,12 +823,12 @@ function startBridge(options) {
         return;
     }
   }
-  async function deliver(seq, target, message) {
+  async function deliver(seq, target, message2) {
     try {
       let refused = null;
       for (; ; ) {
         try {
-          refused = inject(target, message);
+          refused = inject(target, message2);
           break;
         } catch (error) {
           if (error instanceof RelayError2) {
@@ -841,7 +842,7 @@ function startBridge(options) {
           if (attempts >= DEAD_LETTER_ATTEMPTS) {
             await settled("reject", { seq, code: "dead_letter" });
             settle(seq);
-            core.record("dead_letter", null, typeof message?.id === "string" ? message.id : null, { seq, target: String(target), attempts, error: why });
+            core.record("dead_letter", null, typeof message2?.id === "string" ? message2.id : null, { seq, target: String(target), attempts, error: why });
             return;
           }
           await new Promise((resolve) => setTimeout(resolve, injectBackoffMs * 2 ** (attempts - 1)));
@@ -859,20 +860,20 @@ function startBridge(options) {
       log(`relay bridge: delivery ${seq} failed: ${error instanceof Error ? error.message : String(error)}`);
     }
   }
-  function inject(target, message) {
+  function inject(target, message2) {
     const endpoint = core.localTarget(target);
-    if ((hooks.policyError() ?? core.inboundRefusal(endpoint, message)) !== null) return "policy_refused";
-    if (core.admit(endpoint, message).duplicate) return null;
+    if ((hooks.policyError() ?? core.inboundRefusal(endpoint, message2)) !== null) return "policy_refused";
+    if (core.admit(endpoint, message2).duplicate) return null;
     hooks.wake(endpoint);
     hooks.fleet({
       type: "message",
-      machine: message.from.machine ?? null,
-      id: message.id,
-      thread: message.thread,
-      kind: message.kind,
-      from: localAddress(message.from.address),
+      machine: message2.from.machine ?? null,
+      id: message2.id,
+      thread: message2.thread,
+      kind: message2.kind,
+      from: localAddress(message2.from.address),
       to: [core.addressOf(endpoint)],
-      hop: message.hop
+      hop: message2.hop
     });
     return null;
   }
@@ -999,6 +1000,24 @@ function startBridge(options) {
     }
     if (mirrored(view)) fireAndForget("mirror", { endpoints: [outward(view)], full: false });
   }
+  async function mirrorNow(endpoints) {
+    if (!ready) return false;
+    const views = endpoints.flatMap((id) => {
+      try {
+        const view = core.whoami(id);
+        return mirrored(view) ? [outward(view)] : [];
+      } catch {
+        return [];
+      }
+    });
+    if (!views.length) return false;
+    try {
+      for (let i = 0; i < views.length; i += MIRROR_NOW_BATCH) await request("mirror", { endpoints: views.slice(i, i + MIRROR_NOW_BATCH), full: false });
+      return true;
+    } catch {
+      return false;
+    }
+  }
   function renamed(endpoint, was) {
     if (!ready) return;
     let view;
@@ -1054,6 +1073,10 @@ function startBridge(options) {
     if (!connected) throw unavailable();
     return (await request("list", { contacts: true })).contacts;
   }
+  async function machineWake(params) {
+    if (!connected) throw unavailable();
+    return request("machine_wake", params);
+  }
   function status() {
     return { enabled: true, machine, url: options.url, connected, last_error: lastError, since };
   }
@@ -1083,12 +1106,12 @@ function startBridge(options) {
     if (socket && socket.readyState <= 1) socket.close(1e3, "the broker stopped");
   }
   connect();
-  return { forward, changed, resync, renamed, activity, local, contacts, status, relink, stop };
+  return { forward, changed, mirrorNow, resync, renamed, activity, local, contacts, machineWake, status, relink, stop };
 }
 
 // src/server.ts
 import { createHash as createHash4 } from "crypto";
-import { chmodSync as chmodSync2, existsSync as existsSync2, lstatSync as lstatSync2, readFileSync as readFileSync6, realpathSync, rmdirSync, unlinkSync as unlinkSync2 } from "fs";
+import { chmodSync as chmodSync2, existsSync as existsSync2, lstatSync as lstatSync2, readFileSync as readFileSync7, realpathSync, rmdirSync, unlinkSync as unlinkSync2 } from "fs";
 import { createServer } from "net";
 import { homedir as homedir2 } from "os";
 import { dirname as dirname2, isAbsolute, join as join4, sep } from "path";
@@ -1223,10 +1246,199 @@ function openDatabase(path) {
   };
 }
 
+// src/wakers.ts
+import { spawn } from "child_process";
+import { readFileSync as readFileSync6, statSync } from "fs";
+import { WAKE_PROVIDER, wakeDeclineOf } from "@scp-dev/core";
+var TIMEOUT_DEFAULT_S = 60;
+var TIMEOUT_MAX_S = 3600;
+var STDERR_KEEP = 1024;
+var STDOUT_KEEP = 4096;
+var WAKE_DECLINED_EXIT = 77;
+function declineOf(code, stdout) {
+  if (code !== WAKE_DECLINED_EXIT) return null;
+  try {
+    return wakeDeclineOf(JSON.parse(stdout).declined);
+  } catch {
+    return null;
+  }
+}
+function parseWakers(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("wakers is an object of provider names");
+  const out = {};
+  for (const [provider, entry] of Object.entries(value)) {
+    if (!WAKE_PROVIDER.test(provider)) throw new Error(`${JSON.stringify(provider)} is not a provider name`);
+    if (typeof entry !== "object" || entry === null || Array.isArray(entry)) throw new Error(`waker ${provider} is not an object`);
+    const { command, timeout_s } = entry;
+    if (!Array.isArray(command) || !command.length || !command.every((part) => typeof part === "string" && part.length > 0))
+      throw new Error(`waker ${provider}: command is a nonempty array of strings`);
+    const timeout = timeout_s ?? TIMEOUT_DEFAULT_S;
+    if (typeof timeout !== "number" || !Number.isInteger(timeout) || timeout < 1 || timeout > TIMEOUT_MAX_S)
+      throw new Error(`waker ${provider}: timeout_s is a whole number of seconds from 1 to ${TIMEOUT_MAX_S}`);
+    out[provider] = { command, timeout_s: timeout };
+  }
+  return out;
+}
+function startWakers(options) {
+  const busy = /* @__PURE__ */ new Set();
+  const children = /* @__PURE__ */ new Set();
+  let stopped = false;
+  let fixed = null;
+  if (options.wakers !== void 0) {
+    try {
+      fixed = parseWakers(options.wakers);
+    } catch (error) {
+      options.log(`relay broker: the wakers are invalid (${message(error)}); no wake hook runs`);
+      fixed = {};
+    }
+  }
+  let loaded = null;
+  function configs() {
+    if (fixed) return fixed;
+    const stat = statSync(options.path, { throwIfNoEntry: false });
+    if (!stat) {
+      loaded = null;
+      return {};
+    }
+    if (loaded && loaded.mtime === stat.mtimeMs && loaded.size === stat.size) return loaded.wakers;
+    let wakers = {};
+    try {
+      wakers = parseWakers(JSON.parse(readFileSync6(options.path, "utf8")));
+    } catch (error) {
+      options.log(`relay broker: the wakers at ${options.path} are invalid (${message(error)}); no wake hook runs until it is fixed`);
+    }
+    loaded = { mtime: stat.mtimeMs, size: stat.size, wakers };
+    return wakers;
+  }
+  function finish(job, started, result) {
+    busy.delete(job.endpoint);
+    if (stopped) return;
+    try {
+      if (options.core.wakeResult(job.endpoint, { ...result, ms: options.now() - started, message: job.message.id })) options.changed(job.endpoint);
+    } catch (error) {
+      options.log(`relay broker: a wake result was not recorded: ${message(error)}`);
+    }
+  }
+  function exec(config, input, env) {
+    return new Promise((resolve) => {
+      let child;
+      try {
+        child = spawn(config.command[0], config.command.slice(1), { stdio: ["pipe", "pipe", "pipe"], env, detached: true });
+      } catch (error) {
+        resolve({ ok: false, error: message(error), stdout: "" });
+        return;
+      }
+      children.add(child);
+      let stdout = "";
+      let stderr = "";
+      let settled = false;
+      let timedOut = false;
+      const timer = setTimeout(() => {
+        timedOut = true;
+        kill(child);
+      }, config.timeout_s * 1e3);
+      const settle = (result) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        children.delete(child);
+        resolve(result);
+      };
+      child.stdout?.on("data", (chunk) => {
+        if (stdout.length < STDOUT_KEEP) stdout += chunk.toString("utf8").slice(0, STDOUT_KEEP - stdout.length);
+      });
+      child.stderr?.on("data", (chunk) => {
+        if (stderr.length < STDERR_KEEP) stderr += chunk.toString("utf8").slice(0, STDERR_KEEP - stderr.length);
+      });
+      child.on("error", (error) => settle({ ok: false, error: message(error), stdout: "" }));
+      child.on("close", (code, signal) => {
+        if (timedOut) settle({ ok: false, error: `timed out after ${config.timeout_s} s`, stdout: "" });
+        else if (code === 0) settle({ ok: true, code, stdout });
+        else {
+          const last = stderr.trim().split("\n").at(-1)?.trim();
+          settle({ ok: false, code, error: `${code === null ? `killed by ${signal}` : `exit ${code}`}${last ? `: ${last}` : ""}`, stdout });
+        }
+      });
+      child.stdin?.on("error", () => {
+      });
+      child.stdin?.end(input);
+    });
+  }
+  async function run(job, config) {
+    const started = options.now();
+    const event = {
+      scp: 1,
+      event: "wake",
+      endpoint: job.address,
+      handle: job.handle,
+      provider: job.provider,
+      ref: job.ref,
+      machine: options.machine,
+      message: job.message,
+      unread: job.unread
+    };
+    const env = {
+      ...process.env,
+      SCP_DIR: options.dir,
+      SCP_WAKE_ENDPOINT: job.address,
+      SCP_WAKE_HANDLE: job.handle,
+      SCP_WAKE_REF: job.ref ?? "",
+      SCP_WAKE_PROVIDER: job.provider,
+      SCP_WAKE_MESSAGE_ID: job.message.id,
+      SCP_WAKE_FROM: job.message.from ?? "",
+      SCP_WAKE_KIND: job.message.kind,
+      SCP_WAKE_THREAD: job.message.thread
+    };
+    const result = await exec(config, `${JSON.stringify(event)}
+`, env);
+    const declined = result.ok ? null : declineOf(result.code, result.stdout);
+    finish(job, started, { ok: result.ok, code: result.code ?? null, ...declined ? { declined } : { error: result.error ?? null } });
+  }
+  return {
+    known: (provider) => Object.hasOwn(configs(), provider),
+    kick(endpoint) {
+      try {
+        if (stopped || busy.has(endpoint)) return;
+        const job = options.core.wakeDue(endpoint, options.now());
+        if (!job) return;
+        busy.add(endpoint);
+        const config = configs()[job.provider];
+        if (!config) {
+          finish(job, options.now(), { ok: false, error: `no waker ${job.provider} is configured` });
+          return;
+        }
+        run(job, config).catch((error) => {
+          busy.delete(endpoint);
+          options.log(`relay broker: a wake hook failed: ${message(error)}`);
+        });
+      } catch (error) {
+        busy.delete(endpoint);
+        options.log(`relay broker: a wake hook did not start: ${message(error)}`);
+      }
+    },
+    stop() {
+      stopped = true;
+      for (const child of children) kill(child);
+      children.clear();
+    }
+  };
+}
+function kill(child) {
+  try {
+    if (child.pid) process.kill(-child.pid, "SIGKILL");
+  } catch {
+    child.kill("SIGKILL");
+  }
+}
+function message(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+
 // src/server.ts
 var SWEEP_MS = 10 * 6e4;
 var PRESENCE_MS = 6e4;
 var OWNERSHIP_MS = 2e3;
+var HIBERNATE_BARRIER_MS = 1e4;
 var OBSERVER_METHODS = /* @__PURE__ */ new Set(["list", "snapshot", "watch", "log", "policy", "policy_check", "cloud_status", "bridge_relink"]);
 var SCOPE_METHODS = /* @__PURE__ */ new Set([
   "rules",
@@ -1306,6 +1518,7 @@ async function startBroker(options) {
   let core;
   let judge;
   let bridge = null;
+  let wakers = null;
   let listener;
   let socketIno;
   try {
@@ -1388,6 +1601,7 @@ async function startBroker(options) {
       if ((known.get(view.id) ?? "offline") !== view.presence) presenceChanged(view);
       else if (wakes.has(view.id) && wakes.get(view.id) !== view.wake) bridge?.changed(view.id);
       wakes.set(view.id, view.wake);
+      if (view.presence === "hibernated") wakers?.kick(view.id);
     }
     for (const id of known.keys()) if (!live.has(id)) known.delete(id);
     for (const id of wakes.keys()) if (!live.has(id)) wakes.delete(id);
@@ -1403,7 +1617,7 @@ async function startBroker(options) {
   }
   function ownsState() {
     try {
-      return ownsSocket() && readFileSync6(lock.path, "utf8") === lock.content;
+      return ownsSocket() && readFileSync7(lock.path, "utf8") === lock.content;
     } catch {
       return false;
     }
@@ -1419,6 +1633,16 @@ async function startBroker(options) {
   };
   if (options.bridge) bridge = startBridge({ ...options.bridge, core, onRemote: remoteHooks });
   else if (options.link) bridge = options.link({ core, hooks: remoteHooks, send: (from, params, origin) => judge.send(from, params, origin, deliver) });
+  wakers = startWakers({
+    core,
+    path: options.wakersPath ?? join4(dirname2(policyFile), "wakers.json"),
+    wakers: options.wakers,
+    dir,
+    machine: options.machine ?? options.bridge?.machine ?? shortHostname(),
+    now: options.now ?? Date.now,
+    log,
+    changed: (endpoint) => presenceChanged(core.whoami(endpoint))
+  });
   async function relink() {
     const config = options.bridgeConfig ? await options.bridgeConfig() : options.bridge;
     if (!config || bridge && config.url === options.bridge?.url && config.machine === options.bridge?.machine) {
@@ -1436,7 +1660,7 @@ async function startBroker(options) {
   function loadPolicy() {
     let text;
     try {
-      text = readFileSync6(policyFile, "utf8");
+      text = readFileSync7(policyFile, "utf8");
     } catch (error) {
       const code = error.code;
       const dangling = code === "ENOENT" && lstatSync2(policyFile, { throwIfNoEntry: false }) !== void 0;
@@ -1518,7 +1742,15 @@ async function startBroker(options) {
   }
   function presenceChanged(view) {
     known.set(view.id, view.presence);
-    notifyFleet({ type: "presence", endpoint: view.address, presence: view.presence, harness: view.harness, labels: view.labels });
+    notifyFleet({
+      type: "presence",
+      endpoint: view.address,
+      presence: view.presence,
+      harness: view.harness,
+      labels: view.labels,
+      ...view.wake_failed ? { wake_failed: view.wake_failed } : {},
+      ...view.wake_declined ? { wake_declined: view.wake_declined } : {}
+    });
     bridge?.changed(view.id);
   }
   function membership(view, session, joined, was) {
@@ -1599,11 +1831,25 @@ async function startBroker(options) {
       broker: brokerInfo()
     };
   }
+  async function mirrorBarrier(endpoints) {
+    if (!bridge?.mirrorNow || !endpoints.length) return false;
+    let timer;
+    const timeout = new Promise((resolve) => {
+      timer = setTimeout(() => resolve(false), HIBERNATE_BARRIER_MS);
+    });
+    try {
+      return await Promise.race([bridge.mirrorNow(endpoints), timeout]);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
   function wake(endpoint) {
     const set = waiters.get(endpoint);
-    if (!set) return;
-    waiters.delete(endpoint);
-    for (const fn of set) fn();
+    if (set) {
+      waiters.delete(endpoint);
+      for (const fn of set) fn();
+    }
+    wakers?.kick(endpoint);
   }
   function waitFor(endpoint, ms, s) {
     return new Promise((resolve) => {
@@ -1702,8 +1948,8 @@ async function startBroker(options) {
       pump(s);
     }
   }
-  function invalidRequest(s, id, message) {
-    write(s, { jsonrpc: "2.0", id, error: { code: -32600, message, data: { code: "invalid_params" } } });
+  function invalidRequest(s, id, message2) {
+    write(s, { jsonrpc: "2.0", id, error: { code: -32600, message: message2, data: { code: "invalid_params" } } });
   }
   async function answer(s, line) {
     if (closing) return;
@@ -2003,6 +2249,23 @@ async function startBroker(options) {
         bridge?.resync();
         return unlinked;
       }
+      case "hibernate": {
+        const known2 = { known: (provider) => wakers?.known(provider) ?? false };
+        const set = { wake: params.wake, ref: params.ref, eta_s: params.eta_s };
+        if (params.all !== void 0 && params.all !== true) throw new RelayError3("invalid_params", "all is true or absent");
+        if (params.all === true && params.endpoint !== void 0) throw new RelayError3("invalid_params", "all names no endpoint");
+        const result = params.all === true ? core.hibernateAll(actorOf(s), set, known2) : core.hibernate(actorOf(s), { ...params.endpoint !== void 0 ? { endpoint: params.endpoint } : {}, ...set }, known2);
+        const views = "endpoints" in result ? result.endpoints : [result.endpoint];
+        for (const view of views) presenceChanged(view);
+        return { ...result, mirrored: await mirrorBarrier(views.map((view) => view.id)) };
+      }
+      case "machine_wake": {
+        if (actorOf(s).kind !== "human") throw new RelayError3("unauthorized", "only the human sets this machine's waker");
+        if (!bridge?.machineWake) throw new RelayError3("unavailable", "this broker has no hub link: the machine waker lives on the hub");
+        if (params.url !== null && typeof params.url !== "string") throw new RelayError3("invalid_params", "url is an https URL, or null to clear it");
+        if (params.secret !== void 0 && typeof params.secret !== "string") throw new RelayError3("invalid_params", "secret is a string");
+        return bridge.machineWake({ url: params.url, ...params.secret !== void 0 ? { secret: params.secret } : {} });
+      }
       case "leave": {
         const left = core.leaveSession(actorOf(s), params.endpoint !== void 0 ? { endpoint: params.endpoint } : {});
         for (const c of connections) if (c.data.endpoint === left.endpoint.id && !c.data.transient && !c.data.lineage && c !== s) c.end();
@@ -2225,24 +2488,24 @@ async function startBroker(options) {
           })
         );
         if (result.handback) {
-          const message = core.delivered(spawner, result.handback);
+          const message2 = core.delivered(spawner, result.handback);
           wake(spawner);
           bridge?.local?.({
-            id: message.id,
-            thread: message.thread,
+            id: message2.id,
+            thread: message2.thread,
             kind: "handback",
-            from: message.from.endpoint,
+            from: message2.from.endpoint,
             to: [spawner],
             at: (options.now ?? Date.now)()
           });
           notifyFleet({
             type: "message",
-            id: message.id,
-            thread: message.thread,
-            kind: message.kind,
-            from: message.from.address,
-            to: [message.to],
-            hop: message.hop
+            id: message2.id,
+            thread: message2.thread,
+            kind: message2.kind,
+            from: message2.from.address,
+            to: [message2.to],
+            hop: message2.hop
           });
         }
         if (result.endpoint) {
@@ -2286,6 +2549,7 @@ async function startBroker(options) {
       clearInterval(sweeper);
       clearInterval(presenceTimer);
       clearInterval(ownership);
+      wakers?.stop();
       await bridge?.stop();
       if (idleTimer) clearTimeout(idleTimer);
       for (const set of [...waiters.values()]) for (const fn of [...set]) fn();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scp-dev/broker",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "Swarm Communication Protocol local broker: a unix socket JSON-RPC daemon over the relay core, its client and the bridge to an account hub.",
   "license": "Apache-2.0",
   "homepage": "https://scp.dev",
@@ -37,8 +37,8 @@
     "typecheck": "tsc -p ."
   },
   "dependencies": {
-    "@scp-dev/core": "0.4.0",
-    "@scp-dev/hub": "0.6.3"
+    "@scp-dev/core": "0.5.0",
+    "@scp-dev/hub": "0.7.0"
   },
   "devDependencies": {
     "tsup": "^8.5.1"

package/src/bridge.ts

@@ -76,6 +76,8 @@ export interface Bridge {
   forward(from: string, params: SendParams, origin?: { tty?: boolean }): Promise<SendResult>
   /** a local endpoint was claimed, let go or changed its status: mirrored as a delta */
   changed(endpoint: string): void
+  /** mirrors these endpoints now and resolves true once the hub took the delta, false when it did not or none is mirrored */
+  mirrorNow?(endpoints: string[]): Promise<boolean>
   /** rules, caps, groups or a title changed, which can touch several endpoints: every mirrored endpoint is sent again */
   resync(): void
   /** a local endpoint took another handle: the hub renames its mirror in place, keeping what queued for it */
@@ -86,6 +88,8 @@ export interface Bridge {
   local?(event: LocalDelivery): void
   /** the endpoints of other accounts this account's contacts let it reach */
   contacts(): Promise<XListed[]>
+  /** sets (or with a null url clears) the URL the hub calls to wake this machine; the secret signs those calls and is never shown again */
+  machineWake?(params: { url: string | null; secret?: string }): Promise<{ set: boolean }>
   status(): BridgeStatus
   /** reads the credential again and reconnects now, also after the hub revoked or replaced this bridge */
   relink(): void
@@ -102,6 +106,9 @@ export interface LocalDelivery {
   at: number
 }
 
+/** the endpoints one mirrorNow frame carries */
+const MIRROR_NOW_BATCH = 32
+
 /** local deliveries waiting for the next batch; past this, new ones are dropped until it is sent */
 const LOCAL_BUFFER = 1000
 
@@ -639,6 +646,25 @@ export function startBridge(options: BridgeOptions): Bridge {
     if (mirrored(view)) fireAndForget("mirror", { endpoints: [outward(view)], full: false })
   }
 
+  async function mirrorNow(endpoints: string[]): Promise<boolean> {
+    if (!ready) return false
+    const views = endpoints.flatMap((id) => {
+      try {
+        const view = core.whoami(id)
+        return mirrored(view) ? [outward(view)] : []
+      } catch {
+        return []
+      }
+    })
+    if (!views.length) return false
+    try {
+      for (let i = 0; i < views.length; i += MIRROR_NOW_BATCH) await request("mirror", { endpoints: views.slice(i, i + MIRROR_NOW_BATCH), full: false })
+      return true
+    } catch {
+      return false
+    }
+  }
+
   function renamed(endpoint: string, was: string) {
     if (!ready) return
     let view: EndpointView
@@ -700,6 +726,11 @@ export function startBridge(options: BridgeOptions): Bridge {
     return (await request<{ contacts: XListed[] }>("list", { contacts: true })).contacts
   }
 
+  async function machineWake(params: { url: string | null; secret?: string }): Promise<{ set: boolean }> {
+    if (!connected) throw unavailable()
+    return request<{ set: boolean }>("machine_wake", params)
+  }
+
   function status(): BridgeStatus {
     return { enabled: true, machine, url: options.url, connected, last_error: lastError, since }
   }
@@ -732,5 +763,5 @@ export function startBridge(options: BridgeOptions): Bridge {
   }
 
   connect()
-  return { forward, changed, resync, renamed, activity, local, contacts, status, relink, stop }
+  return { forward, changed, mirrorNow, resync, renamed, activity, local, contacts, machineWake, status, relink, stop }
 }

package/src/local-sleep-waker.ts

@@ -0,0 +1,29 @@
+/**
+ * A reference wake hook for tests: it stands in for a provider that resumes a stopped harness on this machine. It reads the wake event on
+ * stdin, claims the endpoint again as the resumed session would (`SCP_WAKE_HANDLE`, against the broker in `SCP_DIR`), drains and acks its
+ * inbox, stays up briefly and exits 0. Usage: bun local-sleep-waker.ts <harness> [<file to append the event and the read ids to>]
+ */
+import { appendFileSync } from "node:fs"
+import { setTimeout as sleep } from "node:timers/promises"
+import type { Envelope } from "@scp-dev/core"
+import { RelayClient } from "./client"
+import { socketPath } from "./paths"
+
+const [harness = "cli", record] = process.argv.slice(2)
+let event = ""
+for await (const chunk of process.stdin) event += String(chunk)
+const dir = process.env.SCP_DIR
+const handle = process.env.SCP_WAKE_HANDLE
+if (!dir || !handle) {
+  console.error("local-sleep-waker: SCP_DIR and SCP_WAKE_HANDLE are required")
+  process.exit(2)
+}
+if (record) appendFileSync(record, event)
+const client = await RelayClient.connect(socketPath(dir))
+await client.hello({ claim: { handle, harness } })
+const { messages } = await client.request<{ messages: Envelope[] }>("inbox", {})
+const seqs = messages.flatMap((m) => (m.seq === undefined ? [] : [m.seq]))
+if (seqs.length) await client.request("ack", { seqs })
+if (record) appendFileSync(record, `${JSON.stringify({ read: messages.map((m) => m.id) })}\n`)
+await sleep(200)
+client.close()

package/src/server.ts

@@ -34,6 +34,7 @@ import { accountHome, accountHomeVerified, policyPath as defaultPolicyPath, priv
 import { writeSessionMarker } from "./sessions"
 import { Conn } from "./socket"
 import { type Database, openDatabase } from "./sqlite"
+import { startWakers, type Wakers } from "./wakers"
 
 export interface BrokerOptions {
   dir: string
@@ -70,6 +71,10 @@ export interface BrokerOptions {
   env?: Record<string, string | undefined> | undefined
   /** the judge model client for a policy's `judge.model` (default the provider's HTTP API) */
   judgeClient?: JudgeClientFactory | undefined
+  /** the wake hooks' configuration file (default wakers.json beside the policy file), read again when it changes */
+  wakersPath?: string | undefined
+  /** the wake hooks by provider, in place of the file */
+  wakers?: Record<string, unknown> | undefined
 }
 
 /** what a link gets from the broker: its core, the remote hooks a bridge gets, and a send as a local endpoint or a mirrored one through the judge */
@@ -131,6 +136,8 @@ interface Connection {
 const SWEEP_MS = 10 * 60_000
 const PRESENCE_MS = 60_000
 const OWNERSHIP_MS = 2_000
+/** how long a hibernate waits for the hub to take its mirror */
+const HIBERNATE_BARRIER_MS = 10_000
 const OBSERVER_METHODS = new Set(["list", "snapshot", "watch", "log", "policy", "policy_check", "cloud_status", "bridge_relink"])
 const SCOPE_METHODS = new Set([
   "rules",
@@ -223,6 +230,7 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
   let core!: FederatedCore
   let judge!: JudgeGate
   let bridge: Bridge | null = null
+  let wakers: Wakers | null = null
   let listener!: Server
   let socketIno: number | bigint | undefined
   try {
@@ -305,6 +313,7 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
       if ((known.get(view.id) ?? "offline") !== view.presence) presenceChanged(view)
       else if (wakes.has(view.id) && wakes.get(view.id) !== view.wake) bridge?.changed(view.id)
       wakes.set(view.id, view.wake)
+      if (view.presence === "hibernated") wakers?.kick(view.id)
     }
     for (const id of known.keys()) if (!live.has(id)) known.delete(id)
     for (const id of wakes.keys()) if (!live.has(id)) wakes.delete(id)
@@ -340,6 +349,16 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
   }
   if (options.bridge) bridge = startBridge({ ...options.bridge, core, onRemote: remoteHooks })
   else if (options.link) bridge = options.link({ core, hooks: remoteHooks, send: (from, params, origin) => judge.send(from, params, origin, deliver) })
+  wakers = startWakers({
+    core,
+    path: options.wakersPath ?? join(dirname(policyFile), "wakers.json"),
+    wakers: options.wakers,
+    dir,
+    machine: options.machine ?? options.bridge?.machine ?? shortHostname(),
+    now: options.now ?? Date.now,
+    log,
+    changed: (endpoint) => presenceChanged(core.whoami(endpoint)),
+  })
 
   /** links the bridge again with the credential on disk; a configuration this broker cannot take without a restart stops it */
   async function relink(): Promise<object> {
@@ -461,7 +480,15 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
   /** an endpoint's presence changed outside a connection's own hello or close: watchers and the other machines hear it */
   function presenceChanged(view: EndpointView) {
     known.set(view.id, view.presence)
-    notifyFleet({ type: "presence", endpoint: view.address, presence: view.presence, harness: view.harness, labels: view.labels })
+    notifyFleet({
+      type: "presence",
+      endpoint: view.address,
+      presence: view.presence,
+      harness: view.harness,
+      labels: view.labels,
+      ...(view.wake_failed ? { wake_failed: view.wake_failed } : {}),
+      ...(view.wake_declined ? { wake_declined: view.wake_declined } : {}),
+    })
     bridge?.changed(view.id)
   }
 
@@ -558,11 +585,28 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
     }
   }
 
+  /** whether the hub took these endpoints' mirror within HIBERNATE_BARRIER_MS, so the machine may be suspended at once */
+  async function mirrorBarrier(endpoints: string[]): Promise<boolean> {
+    if (!bridge?.mirrorNow || !endpoints.length) return false
+    let timer: ReturnType<typeof setTimeout> | undefined
+    const timeout = new Promise<boolean>((resolve) => {
+      timer = setTimeout(() => resolve(false), HIBERNATE_BARRIER_MS)
+    })
+    try {
+      return await Promise.race([bridge.mirrorNow(endpoints), timeout])
+    } finally {
+      clearTimeout(timer)
+    }
+  }
+
+  /** a delivery reached `endpoint`: its waiters return, and a hibernated one's wake hook runs when due */
   function wake(endpoint: string) {
     const set = waiters.get(endpoint)
-    if (!set) return
-    waiters.delete(endpoint)
-    for (const fn of set) fn()
+    if (set) {
+      waiters.delete(endpoint)
+      for (const fn of set) fn()
+    }
+    wakers?.kick(endpoint)
   }
 
   function waitFor(endpoint: string, ms: number, s: Conn<Connection>): Promise<void> {
@@ -993,6 +1037,26 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
         bridge?.resync()
         return unlinked
       }
+      case "hibernate": {
+        const known = { known: (provider: string) => wakers?.known(provider) ?? false }
+        const set = { wake: params.wake, ref: params.ref, eta_s: params.eta_s }
+        if (params.all !== undefined && params.all !== true) throw new RelayError("invalid_params", "all is true or absent")
+        if (params.all === true && params.endpoint !== undefined) throw new RelayError("invalid_params", "all names no endpoint")
+        const result =
+          params.all === true
+            ? core.hibernateAll(actorOf(s), set, known)
+            : core.hibernate(actorOf(s), { ...(params.endpoint !== undefined ? { endpoint: params.endpoint } : {}), ...set }, known)
+        const views = "endpoints" in result ? result.endpoints : [result.endpoint]
+        for (const view of views) presenceChanged(view)
+        return { ...result, mirrored: await mirrorBarrier(views.map((view) => view.id)) }
+      }
+      case "machine_wake": {
+        if (actorOf(s).kind !== "human") throw new RelayError("unauthorized", "only the human sets this machine's waker")
+        if (!bridge?.machineWake) throw new RelayError("unavailable", "this broker has no hub link: the machine waker lives on the hub")
+        if (params.url !== null && typeof params.url !== "string") throw new RelayError("invalid_params", "url is an https URL, or null to clear it")
+        if (params.secret !== undefined && typeof params.secret !== "string") throw new RelayError("invalid_params", "secret is a string")
+        return bridge.machineWake({ url: params.url, ...(params.secret !== undefined ? { secret: params.secret } : {}) })
+      }
       case "leave": {
         const left = core.leaveSession(actorOf(s), params.endpoint !== undefined ? { endpoint: params.endpoint } : {})
         for (const c of connections) if (c.data.endpoint === left.endpoint.id && !c.data.transient && !c.data.lineage && c !== s) c.end()
@@ -1292,6 +1356,7 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
       clearInterval(sweeper)
       clearInterval(presenceTimer)
       clearInterval(ownership)
+      wakers?.stop()
       await bridge?.stop()
       if (idleTimer) clearTimeout(idleTimer)
       for (const set of [...waiters.values()]) for (const fn of [...set]) fn()

package/src/wakers.ts

@@ -0,0 +1,232 @@
+import { type ChildProcess, spawn } from "node:child_process"
+import { readFileSync, statSync } from "node:fs"
+import { WAKE_PROVIDER, type WakeDecline, type WakeJob, wakeDeclineOf } from "@scp-dev/core"
+import type { FederatedCore } from "@scp-dev/hub"
+
+export interface WakerConfig {
+  command: string[]
+  timeout_s: number
+}
+
+export type WakerConfigs = Record<string, WakerConfig>
+
+const TIMEOUT_DEFAULT_S = 60
+const TIMEOUT_MAX_S = 3600
+const STDERR_KEEP = 1024
+const STDOUT_KEEP = 4096
+/** a waker's exit code for a decline, with `{"declined": "<reason>"}` on stdout */
+export const WAKE_DECLINED_EXIT = 77
+
+/** the decline a waker's exit 77 and stdout carry, or null */
+function declineOf(code: number | null | undefined, stdout: string): WakeDecline | null {
+  if (code !== WAKE_DECLINED_EXIT) return null
+  try {
+    return wakeDeclineOf((JSON.parse(stdout) as { declined?: unknown }).declined)
+  } catch {
+    return null
+  }
+}
+
+/** a wakers.json object: `{ "<provider>": { "command": [argv...], "timeout_s"?: N } }`, other fields ignored; throws on anything else */
+export function parseWakers(value: unknown): WakerConfigs {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("wakers is an object of provider names")
+  const out: WakerConfigs = {}
+  for (const [provider, entry] of Object.entries(value as Record<string, unknown>)) {
+    if (!WAKE_PROVIDER.test(provider)) throw new Error(`${JSON.stringify(provider)} is not a provider name`)
+    if (typeof entry !== "object" || entry === null || Array.isArray(entry)) throw new Error(`waker ${provider} is not an object`)
+    const { command, timeout_s } = entry as { command?: unknown; timeout_s?: unknown }
+    if (!Array.isArray(command) || !command.length || !command.every((part) => typeof part === "string" && part.length > 0))
+      throw new Error(`waker ${provider}: command is a nonempty array of strings`)
+    const timeout = timeout_s ?? TIMEOUT_DEFAULT_S
+    if (typeof timeout !== "number" || !Number.isInteger(timeout) || timeout < 1 || timeout > TIMEOUT_MAX_S)
+      throw new Error(`waker ${provider}: timeout_s is a whole number of seconds from 1 to ${TIMEOUT_MAX_S}`)
+    out[provider] = { command: command as string[], timeout_s: timeout }
+  }
+  return out
+}
+
+export interface Wakers {
+  /** whether a waker is configured for `provider` */
+  known(provider: string): boolean
+  /** runs the endpoint's wake hook when one is due; never throws, never waits */
+  kick(endpoint: string): void
+  /** kills the hooks still running */
+  stop(): void
+}
+
+type RunResult = { ok: boolean; code?: number | null; error?: string; stdout: string }
+type Outcome = { ok: boolean; code?: number | null; error?: string | null; declined?: WakeDecline }
+
+/**
+ * the broker's wake hooks: `wakers` as given, else the file at `path`, read again when its mtime changes. A malformed file is logged once
+ * and configures nothing. At most one hook runs per endpoint; `changed` hears an endpoint whose shown wake failure changed
+ */
+export function startWakers(options: {
+  core: FederatedCore
+  path: string
+  wakers?: Record<string, unknown> | undefined
+  dir: string
+  machine: string
+  now: () => number
+  log: (line: string) => void
+  changed: (endpoint: string) => void
+}): Wakers {
+  const busy = new Set<string>()
+  const children = new Set<ChildProcess>()
+  let stopped = false
+  let fixed: WakerConfigs | null = null
+  if (options.wakers !== undefined) {
+    try {
+      fixed = parseWakers(options.wakers)
+    } catch (error) {
+      options.log(`relay broker: the wakers are invalid (${message(error)}); no wake hook runs`)
+      fixed = {}
+    }
+  }
+  let loaded: { mtime: number; size: number; wakers: WakerConfigs } | null = null
+
+  function configs(): WakerConfigs {
+    if (fixed) return fixed
+    const stat = statSync(options.path, { throwIfNoEntry: false })
+    if (!stat) {
+      loaded = null
+      return {}
+    }
+    if (loaded && loaded.mtime === stat.mtimeMs && loaded.size === stat.size) return loaded.wakers
+    let wakers: WakerConfigs = {}
+    try {
+      wakers = parseWakers(JSON.parse(readFileSync(options.path, "utf8")))
+    } catch (error) {
+      options.log(`relay broker: the wakers at ${options.path} are invalid (${message(error)}); no wake hook runs until it is fixed`)
+    }
+    loaded = { mtime: stat.mtimeMs, size: stat.size, wakers }
+    return wakers
+  }
+
+  function finish(job: WakeJob, started: number, result: Outcome) {
+    busy.delete(job.endpoint)
+    if (stopped) return
+    try {
+      if (options.core.wakeResult(job.endpoint, { ...result, ms: options.now() - started, message: job.message.id })) options.changed(job.endpoint)
+    } catch (error) {
+      options.log(`relay broker: a wake result was not recorded: ${message(error)}`)
+    }
+  }
+
+  /** runs the waker with stdin `input`, keeping the head of its stdout for a decline */
+  function exec(config: WakerConfig, input: string, env: Record<string, string | undefined>): Promise<RunResult> {
+    return new Promise((resolve) => {
+      let child: ChildProcess
+      try {
+        child = spawn(config.command[0]!, config.command.slice(1), { stdio: ["pipe", "pipe", "pipe"], env, detached: true })
+      } catch (error) {
+        resolve({ ok: false, error: message(error), stdout: "" })
+        return
+      }
+      children.add(child)
+      let stdout = ""
+      let stderr = ""
+      let settled = false
+      let timedOut = false
+      const timer = setTimeout(() => {
+        timedOut = true
+        kill(child)
+      }, config.timeout_s * 1000)
+      const settle = (result: RunResult) => {
+        if (settled) return
+        settled = true
+        clearTimeout(timer)
+        children.delete(child)
+        resolve(result)
+      }
+      child.stdout?.on("data", (chunk: Buffer) => {
+        if (stdout.length < STDOUT_KEEP) stdout += chunk.toString("utf8").slice(0, STDOUT_KEEP - stdout.length)
+      })
+      child.stderr?.on("data", (chunk: Buffer) => {
+        if (stderr.length < STDERR_KEEP) stderr += chunk.toString("utf8").slice(0, STDERR_KEEP - stderr.length)
+      })
+      child.on("error", (error) => settle({ ok: false, error: message(error), stdout: "" }))
+      child.on("close", (code, signal) => {
+        if (timedOut) settle({ ok: false, error: `timed out after ${config.timeout_s} s`, stdout: "" })
+        else if (code === 0) settle({ ok: true, code, stdout })
+        else {
+          const last = stderr.trim().split("\n").at(-1)?.trim()
+          settle({ ok: false, code, error: `${code === null ? `killed by ${signal}` : `exit ${code}`}${last ? `: ${last}` : ""}`, stdout })
+        }
+      })
+      child.stdin?.on("error", () => {})
+      child.stdin?.end(input)
+    })
+  }
+
+  async function run(job: WakeJob, config: WakerConfig) {
+    const started = options.now()
+    const event = {
+      scp: 1,
+      event: "wake",
+      endpoint: job.address,
+      handle: job.handle,
+      provider: job.provider,
+      ref: job.ref,
+      machine: options.machine,
+      message: job.message,
+      unread: job.unread,
+    }
+    const env: Record<string, string | undefined> = {
+      ...process.env,
+      SCP_DIR: options.dir,
+      SCP_WAKE_ENDPOINT: job.address,
+      SCP_WAKE_HANDLE: job.handle,
+      SCP_WAKE_REF: job.ref ?? "",
+      SCP_WAKE_PROVIDER: job.provider,
+      SCP_WAKE_MESSAGE_ID: job.message.id,
+      SCP_WAKE_FROM: job.message.from ?? "",
+      SCP_WAKE_KIND: job.message.kind,
+      SCP_WAKE_THREAD: job.message.thread,
+    }
+    const result = await exec(config, `${JSON.stringify(event)}\n`, env)
+    const declined = result.ok ? null : declineOf(result.code, result.stdout)
+    finish(job, started, { ok: result.ok, code: result.code ?? null, ...(declined ? { declined } : { error: result.error ?? null }) })
+  }
+
+  return {
+    known: (provider) => Object.hasOwn(configs(), provider),
+    kick(endpoint) {
+      try {
+        if (stopped || busy.has(endpoint)) return
+        const job = options.core.wakeDue(endpoint, options.now())
+        if (!job) return
+        busy.add(endpoint)
+        const config = configs()[job.provider]
+        if (!config) {
+          finish(job, options.now(), { ok: false, error: `no waker ${job.provider} is configured` })
+          return
+        }
+        run(job, config).catch((error: unknown) => {
+          busy.delete(endpoint)
+          options.log(`relay broker: a wake hook failed: ${message(error)}`)
+        })
+      } catch (error) {
+        busy.delete(endpoint)
+        options.log(`relay broker: a wake hook did not start: ${message(error)}`)
+      }
+    },
+    stop() {
+      stopped = true
+      for (const child of children) kill(child)
+      children.clear()
+    },
+  }
+}
+
+function kill(child: ChildProcess) {
+  try {
+    if (child.pid) process.kill(-child.pid, "SIGKILL")
+  } catch {
+    child.kill("SIGKILL")
+  }
+}
+
+function message(error: unknown): string {
+  return error instanceof Error ? error.message : String(error)
+}