@astrofoundry/pi-astro 0.19.3 → 0.20.0

package/specialists/entry/remote/spc-backup-arcane-entry.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Forced command for the spc-backup key on VM 100 (user arcane, passwordless sudo).
+# authorized_keys line:
+#   command="/usr/local/sbin/spc-backup-arcane-entry",no-port-forwarding,no-agent-forwarding,no-pty,no-X11-forwarding ssh-ed25519 AAAA... spc-backup-arcane@cortex
+set -eu
+cmd="${SSH_ORIGINAL_COMMAND:-}"
+name="${cmd%% *}"
+rest=""
+[ "$cmd" != "$name" ] && rest="${cmd#* }"
+since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
+unit_status() { systemctl show "$1" -p Result -p ExecMainStatus -p ExecMainStartTimestamp -p ExecMainExitTimestamp; systemctl list-timers "$2" --no-pager --all; }
+
+case "$name" in
+  arcane-backup-status) unit_status arcane-backup.service arcane-backup.timer ;;
+  arcane-backup-run) sudo -n systemctl start arcane-backup.service; unit_status arcane-backup.service arcane-backup.timer ;;
+  arcane-backup-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; sudo -n journalctl -u arcane-backup.service --since "$rest" --no-pager -n 200 ;;
+  restic-check-status) unit_status restic-check.service restic-check.timer ;;
+  restic-check-run) sudo -n systemctl start restic-check.service; unit_status restic-check.service restic-check.timer ;;
+  restic-check-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; sudo -n journalctl -u restic-check.service --since "$rest" --no-pager -n 200 ;;
+  *) echo "refused: $cmd" >&2; exit 2 ;;
+esac

package/specialists/entry/remote/spc-backup-pulsar-entry.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+# Forced command for the spc-backup key on Pulsar (root).
+# authorized_keys line:
+#   command="/usr/local/sbin/spc-backup-pulsar-entry",no-port-forwarding,no-agent-forwarding,no-pty,no-X11-forwarding ssh-ed25519 AAAA... spc-backup-pulsar@cortex
+set -eu
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export HOME=/root
+export RESTIC_REPOSITORY=/mnt/pve/unas-backup/restic
+export RESTIC_PASSWORD_FILE=/root/.restic-pw
+cmd="${SSH_ORIGINAL_COMMAND:-}"
+name="${cmd%% *}"
+rest=""
+[ "$cmd" != "$name" ] && rest="${cmd#* }"
+since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
+tag_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9._-]{1,40}$'; }
+snap_ok() { printf '%s' "$1" | grep -Eq '^([0-9a-f]{8,64}|latest)$'; }
+path_ok() { printf '%s' "$1" | grep -Eq '^/[A-Za-z0-9._/-]{1,200}$' && ! printf '%s' "$1" | grep -q '\.\.'; }
+restore_root=/var/tmp/spc-restore
+unit_status() { systemctl show "$1" -p Result -p ExecMainStatus -p ExecMainStartTimestamp -p ExecMainExitTimestamp; systemctl list-timers "$2" --no-pager --all; }
+
+case "$name" in
+  vzdump-jobs) pvesh get /cluster/backup --output-format json ;;
+  vzdump-archives) pvesh get /nodes/pulsar/storage/unas-backup/content --content backup --output-format json ;;
+  vzdump-tasks) pvesh get /nodes/pulsar/tasks --typefilter vzdump --limit 20 --output-format json ;;
+  vzdump-run)
+    case "$rest" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
+    vzdump "$rest" --storage unas-backup --mode snapshot --compress zstd ;;
+  host-backup-status) unit_status pulsar-config-backup.service pulsar-config-backup.timer ;;
+  host-backup-run) systemctl start pulsar-config-backup.service; unit_status pulsar-config-backup.service pulsar-config-backup.timer ;;
+  host-backup-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl -u pulsar-config-backup.service --since "$rest" --no-pager -n 200 ;;
+  restic-snapshots)
+    if [ -n "$rest" ]; then tag_ok "$rest" || { echo "bad tag" >&2; exit 2; }; restic snapshots --json --no-lock --tag "$rest"; else restic snapshots --json --no-lock; fi ;;
+  restic-stats)
+    if [ -n "$rest" ]; then tag_ok "$rest" || { echo "bad tag" >&2; exit 2; }; restic stats --json --no-lock --tag "$rest" latest; else restic stats --json --no-lock latest; fi ;;
+  restic-ls)
+    snap="${rest%% *}"; sub=""; [ "$rest" != "$snap" ] && sub="${rest#* }"
+    snap_ok "$snap" || { echo "bad snapshot" >&2; exit 2; }
+    if [ -n "$sub" ]; then path_ok "$sub" || { echo "bad path" >&2; exit 2; }; restic ls --json --no-lock "$snap" "$sub"; else restic ls --json --no-lock "$snap"; fi ;;
+  restic-restore)
+    snap="${rest%% *}"; sub="${rest#* }"
+    snap_ok "$snap" && path_ok "$sub" || { echo "bad snapshot or path" >&2; exit 2; }
+    install -d -o root -g root -m 0700 "$restore_root/$snap"
+    restic restore --no-lock "$snap" --target "$restore_root/$snap" --include "$sub"
+    find "$restore_root/$snap" -maxdepth 6 | head -n 200 ;;
+  restore-ls) [ -d "$restore_root" ] && find "$restore_root" -maxdepth 4 | head -n 500 || echo "nothing restored" ;;
+  restore-clean) rm -rf "$restore_root"; echo "removed $restore_root" ;;
+  gcs-status) pct exec 102 -- systemctl show gcs-backup.service -p Result -p ExecMainStatus -p ExecMainStartTimestamp -p ExecMainExitTimestamp; pct exec 102 -- systemctl list-timers gcs-backup.timer --no-pager --all ;;
+  gcs-run) pct exec 102 -- systemctl start gcs-backup.service; pct exec 102 -- systemctl show gcs-backup.service -p Result -p ExecMainStatus -p ExecMainExitTimestamp ;;
+  gcs-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; pct exec 102 -- journalctl -u gcs-backup.service --since "$rest" --no-pager -n 200 ;;
+  *) echo "refused: $cmd" >&2; exit 2 ;;
+esac

package/specialists/entry/remote/spc-inference-hermes-entry.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+# Forced command for the spc-inference key on LXC 101 (user hermes, no sudo).
+# authorized_keys line in /home/hermes/.ssh/authorized_keys:
+#   command="/usr/local/sbin/spc-inference-hermes-entry",no-port-forwarding,no-agent-forwarding,no-pty,no-X11-forwarding ssh-ed25519 AAAA... spc-inference-hermes@cortex
+set -eu
+export PATH="$HOME/.local/bin:$HOME/.hermes/hermes-agent/venv/bin:/usr/local/bin:/usr/bin:/bin"
+export XDG_RUNTIME_DIR="/run/user/$(id -u)"
+cmd="${SSH_ORIGINAL_COMMAND:-}"
+name="${cmd%% *}"
+rest=""
+[ "$cmd" != "$name" ] && rest="${cmd#* }"
+since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
+units="hermes-gateway.service hermes-gateway-api.service"
+
+case "$name" in
+  hermes-status) systemctl --user status $units --no-pager --lines=5 || true ;;
+  hermes-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl --user -u hermes-gateway.service -u hermes-gateway-api.service --since "$rest" --no-pager -n 200 ;;
+  hermes-errors) hermes logs errors ;;
+  hermes-restart)
+    case "$rest" in gateway) u=hermes-gateway.service ;; api) u=hermes-gateway-api.service ;; *) echo "bad service" >&2; exit 2 ;; esac
+    systemctl --user restart "$u"; sleep 3; systemctl --user status "$u" --no-pager --lines=5 || true ;;
+  hermes-api-health)
+    printf 'health=%s\n' "$(curl -s -o /dev/null -w '%{http_code}' -m 5 http://localhost:8650/health || echo 000)"
+    printf 'models=%s (401 expected)\n' "$(curl -s -o /dev/null -w '%{http_code}' -m 5 http://localhost:8650/v1/models || echo 000)" ;;
+  hermes-version) hermes --version | head -1 ;;
+  *) echo "refused: $cmd" >&2; exit 2 ;;
+esac

package/specialists/entry/remote/spc-inference-nexus-entry.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Forced command for the spc-inference key on Nexus (macOS, user nexus).
+# authorized_keys line in /Users/nexus/.ssh/authorized_keys:
+#   command="/usr/local/sbin/spc-inference-nexus-entry",no-port-forwarding,no-agent-forwarding,no-pty,no-X11-forwarding ssh-ed25519 AAAA... spc-inference-nexus@cortex
+set -eu
+export PATH=/usr/bin:/bin:/usr/sbin:/sbin
+cmd="${SSH_ORIGINAL_COMMAND:-}"
+name="${cmd%% *}"
+rest=""
+[ "$cmd" != "$name" ] && rest="${cmd#* }"
+uid="$(id -u)"
+label() { case "$1" in embed) echo com.nexus.llama-embed ;; rerank) echo com.nexus.llama-rerank ;; *) return 1 ;; esac; }
+port() { case "$1" in embed) echo 8081 ;; rerank) echo 8082 ;; esac; }
+agent_state() {
+  l="$(label "$1")"
+  printf '%-8s ' "$1"
+  launchctl print "gui/$uid/$l" 2>/dev/null | grep -E '^\s*(state|pid) = ' | tr -s ' ' | tr '\n' ' ' || printf 'not loaded '
+  printf 'health=%s\n' "$(curl -s -o /dev/null -w '%{http_code}' -m 5 "http://127.0.0.1:$(port "$1")/health" || echo 000)"
+}
+
+case "$name" in
+  nexus-status) agent_state embed; agent_state rerank ;;
+  nexus-restart) l="$(label "$rest")" || { echo "bad service" >&2; exit 2; }; launchctl kickstart -k "gui/$uid/$l"; sleep 3; agent_state "$rest" ;;
+  nexus-start)
+    l="$(label "$rest")" || { echo "bad service" >&2; exit 2; }
+    plist="$HOME/Library/LaunchAgents/$l.plist"
+    [ -f "$plist" ] || { echo "missing $plist" >&2; exit 1; }
+    launchctl bootstrap "gui/$uid" "$plist"; sleep 3; agent_state "$rest" ;;
+  nexus-stop) l="$(label "$rest")" || { echo "bad service" >&2; exit 2; }; launchctl bootout "gui/$uid/$l"; agent_state "$rest" ;;
+  nexus-log)
+    svc="${rest%% *}"; lines="${rest#* }"
+    label "$svc" >/dev/null || { echo "bad service" >&2; exit 2; }
+    printf '%s' "$lines" | grep -Eq '^[1-9][0-9]{0,3}$' || { echo "bad lines" >&2; exit 2; }
+    tail -n "$lines" "$HOME/llama-$svc.log" ;;
+  nexus-disk) df -h / ;;
+  *) echo "refused: $cmd" >&2; exit 2 ;;
+esac

package/specialists/entry/remote/spc-inference-pulsar-entry.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+# Forced command for the spc-inference key on Pulsar (root): Hermes guest operations.
+# authorized_keys line:
+#   command="/usr/local/sbin/spc-inference-pulsar-entry",no-port-forwarding,no-agent-forwarding,no-pty,no-X11-forwarding ssh-ed25519 AAAA... spc-inference-pulsar@cortex
+set -eu
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+cmd="${SSH_ORIGINAL_COMMAND:-}"
+
+case "$cmd" in
+  hermes-guest-status) pct status 101 ;;
+  hermes-snapshots) pct listsnapshot 101 ;;
+  hermes-upgrade) /usr/local/bin/hermes-upgrade ;;
+  *) echo "refused: $cmd" >&2; exit 2 ;;
+esac

package/specialists/entry/remote/spc-proxmox-pulsar-entry.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+# Forced command for the spc-proxmox key on Pulsar (root): host facts and fixed reads inside guests.
+# authorized_keys line:
+#   command="/usr/local/sbin/spc-proxmox-pulsar-entry",no-port-forwarding,no-agent-forwarding,no-pty,no-X11-forwarding ssh-ed25519 AAAA... spc-proxmox-pulsar@cortex
+set -eu
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+cmd="${SSH_ORIGINAL_COMMAND:-}"
+name="${cmd%% *}"
+rest=""
+[ "$cmd" != "$name" ] && rest="${cmd#* }"
+since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
+
+guest_cmd() {
+  case "$1" in
+    status) echo 'systemctl --failed --no-pager; uptime' ;;
+    journal) echo 'journalctl -p warning --since "24 hours ago" --no-pager -n 100' ;;
+    df) echo 'df -h -x tmpfs -x devtmpfs' ;;
+    updates) echo 'apt list --upgradable 2>/dev/null' ;;
+    *) return 1 ;;
+  esac
+}
+
+case "$name" in
+  host-status)
+    pveversion
+    uptime
+    free -h
+    df -h / /var/lib/vz
+    sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true ;;
+  host-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl -p warning --since "$rest" --no-pager -n 200 ;;
+  updates) apt list --upgradable 2>/dev/null ;;
+  guest-exec)
+    vmid="${rest%% *}"; action="${rest#* }"
+    case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
+    inner="$(guest_cmd "$action")" || { echo "bad action" >&2; exit 2; }
+    if pct status "$vmid" >/dev/null 2>&1; then
+      pct exec "$vmid" -- sh -c "$inner"
+    elif qm status "$vmid" >/dev/null 2>&1; then
+      qm guest exec "$vmid" --timeout 60 -- sh -c "$inner"
+    else
+      echo "no such guest" >&2; exit 2
+    fi ;;
+  *) echo "refused: $cmd" >&2; exit 2 ;;
+esac

package/specialists/inference/run.ts

@@ -0,0 +1,167 @@
+import { join } from "node:path";
+import { readConfig } from "../lib/config.ts";
+import { ServiceError, UsageError } from "../lib/errors.ts";
+import { main } from "../lib/main.ts";
+import { printJson, printRaw } from "../lib/output.ts";
+import { specialistsHome } from "../lib/paths.ts";
+import { run } from "../lib/proc.ts";
+import { secretPath } from "../lib/secrets.ts";
+import { sshFixed } from "../lib/ssh.ts";
+
+const SERVICE = "inference";
+
+interface InferenceConfig extends Record<string, string> {
+    nexusHost: string;
+    nexusUser: string;
+    hermesHost: string;
+    hermesUser: string;
+    pulsarHost: string;
+    pulsarUser: string;
+    europaHealthUrl: string;
+    /** GitHub repository whose releases carry the Hermes upgrade notes, owner/name. */
+    hermesRepo: string;
+}
+
+const SHAPE = {
+    nexusHost: "string",
+    nexusUser: "string",
+    hermesHost: "string",
+    hermesUser: "string",
+    pulsarHost: "string",
+    pulsarUser: "string",
+    europaHealthUrl: "string",
+    hermesRepo: "string",
+} as const;
+
+type Target = "nexus" | "hermes" | "pulsar";
+
+interface RemoteSpec {
+    target: Target;
+    args: number;
+    help: string;
+    timeoutMs: number;
+    confirm?: true;
+}
+
+const MINUTE = 60_000;
+
+/** Fixed remote commands; the forced commands on Nexus (nexus), LXC 101 (hermes), and Pulsar (root) accept exactly these names. */
+export const INFERENCE_COMMANDS: Readonly<Record<string, RemoteSpec>> = {
+    "nexus-status": { target: "nexus", args: 0, help: "LaunchAgent states and /health codes of the embed and rerank servers", timeoutMs: MINUTE },
+    "nexus-restart": { target: "nexus", args: 1, help: "nexus-restart <embed|rerank>   launchctl kickstart -k", timeoutMs: 2 * MINUTE },
+    "nexus-start": { target: "nexus", args: 1, help: "nexus-start <embed|rerank>     launchctl bootstrap from the tracked plist", timeoutMs: 2 * MINUTE },
+    "nexus-stop": { target: "nexus", args: 1, help: "nexus-stop <embed|rerank> --confirm   launchctl bootout", timeoutMs: 2 * MINUTE, confirm: true },
+    "nexus-log": { target: "nexus", args: 2, help: "nexus-log <embed|rerank> <lines>", timeoutMs: MINUTE },
+    "nexus-disk": { target: "nexus", args: 0, help: "free space on the Nexus disk", timeoutMs: MINUTE },
+    "hermes-status": { target: "hermes", args: 0, help: "both gateway user services", timeoutMs: MINUTE },
+    "hermes-journal": { target: "hermes", args: 1, help: "hermes-journal <since>         user journal of both gateways", timeoutMs: MINUTE },
+    "hermes-errors": { target: "hermes", args: 0, help: "hermes logs errors", timeoutMs: MINUTE },
+    "hermes-restart": { target: "hermes", args: 1, help: "hermes-restart <gateway|api>", timeoutMs: 2 * MINUTE },
+    "hermes-api-health": { target: "hermes", args: 0, help: "/health and /v1/models codes on localhost:8650 (expect 200 and 401)", timeoutMs: MINUTE },
+    "hermes-version": { target: "hermes", args: 0, help: "installed Hermes version", timeoutMs: MINUTE },
+    "hermes-guest-status": { target: "pulsar", args: 0, help: "pct status 101", timeoutMs: MINUTE },
+    "hermes-snapshots": { target: "pulsar", args: 0, help: "pct listsnapshot 101", timeoutMs: MINUTE },
+    "hermes-upgrade": { target: "pulsar", args: 0, help: "hermes-upgrade --confirm       snapshot LXC 101, hermes update, restart, health check", timeoutMs: 20 * MINUTE, confirm: true },
+};
+
+const NEXUS_SERVICES = ["embed", "rerank"];
+const HERMES_SERVICES = ["gateway", "api"];
+const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
+
+const HELP = `inference specialist (llama.cpp on Nexus, Hermes on LXC 101, Europa health)
+
+  <command> [args]            fixed operations through forced-command SSH keys
+${Object.entries(INFERENCE_COMMANDS)
+    .map(([name, c]) => `      ${name.padEnd(22)} ${c.help}`)
+    .join("\n")}
+  europa-health               GET the Europa embedding /health over HTTP (Europa is usually off by design)
+  hermes-releases [n]         latest n upstream Hermes releases with notes (default 5); read them before hermes-upgrade
+
+Europa control and Hermes configuration are operator tasks.`;
+
+export function buildInferenceRemote(args: string[]): { remote: string; spec: RemoteSpec } {
+    const [name, ...given] = args;
+    const spec = name === undefined ? undefined : INFERENCE_COMMANDS[name];
+    if (!spec) throw new UsageError(`unknown command: ${name ?? "(none)"}`);
+    let rest = given;
+    if (spec.confirm) {
+        const index = rest.indexOf("--confirm");
+        if (index === -1) throw new UsageError(`${name} interrupts a service; add --confirm when the task asks for it explicitly`);
+        rest = rest.filter((_, i) => i !== index);
+    }
+    if (rest.length !== spec.args) throw new UsageError(`${name} takes ${spec.args} argument(s)`);
+    if (name.startsWith("nexus-") && spec.args >= 1 && !NEXUS_SERVICES.includes(rest[0])) throw new UsageError(`service must be one of ${NEXUS_SERVICES.join(", ")}`);
+    if (name === "nexus-log") {
+        const n = Number(rest[1]);
+        if (!Number.isInteger(n) || n < 1 || n > 2000) throw new UsageError("lines must be an integer from 1 to 2000");
+    }
+    if (name === "hermes-restart" && !HERMES_SERVICES.includes(rest[0])) throw new UsageError(`service must be one of ${HERMES_SERVICES.join(", ")}`);
+    if (name === "hermes-journal" && !SINCE.test(rest[0])) throw new UsageError("since: letters, digits, spaces, colon, plus, minus only");
+    return { remote: [name, ...rest].join(" "), spec };
+}
+
+interface GithubRelease {
+    tag_name?: string;
+    name?: string;
+    published_at?: string;
+    html_url?: string;
+    prerelease?: boolean;
+    body?: string;
+}
+
+/** Trims GitHub release objects to what an upgrade decision needs. */
+export function compactReleases(releases: GithubRelease[]): { tag: string; name: string; published: string; prerelease: boolean; url: string; notes: string }[] {
+    return releases.map((r) => ({
+        tag: r.tag_name ?? "?",
+        name: r.name ?? "",
+        published: r.published_at ?? "",
+        prerelease: r.prerelease === true,
+        url: r.html_url ?? "",
+        notes: (r.body ?? "").slice(0, 6000),
+    }));
+}
+
+async function hermesReleases(config: InferenceConfig, countArg: string | undefined): Promise<number> {
+    const n = countArg === undefined ? 5 : Number(countArg);
+    if (!Number.isInteger(n) || n < 1 || n > 30) throw new UsageError("hermes-releases [n]: n must be 1..30");
+    if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(config.hermesRepo)) throw new ServiceError("hermesRepo must be owner/name");
+    const response = await fetch(`https://api.github.com/repos/${config.hermesRepo}/releases?per_page=${n}`, {
+        headers: { Accept: "application/vnd.github+json", "User-Agent": "pi-astro-inference-specialist" },
+        signal: AbortSignal.timeout(30_000),
+    });
+    if (!response.ok) throw new ServiceError(`GitHub answered ${response.status} for ${config.hermesRepo} releases`);
+    printJson({ repository: config.hermesRepo, releases: compactReleases((await response.json()) as GithubRelease[]) });
+    return 0;
+}
+
+async function europaHealth(config: InferenceConfig): Promise<number> {
+    const result = await run("/usr/bin/curl", ["--silent", "--output", "/dev/null", "--max-time", "5", "--write-out", "%{http_code}", config.europaHealthUrl], { timeoutMs: 15_000 });
+    const httpCode = result.stdout.trim();
+    const ok = result.code === 0 && httpCode === "200";
+    printJson({ ok, url: config.europaHealthUrl, httpCode: httpCode || null, curlExit: result.code, note: ok ? "Europa is up" : "Europa is off or unreachable; it is off by design most of the time" });
+    return 0;
+}
+
+export async function command(args: string[]): Promise<number> {
+    if (args.length === 0 || args[0] === "--help" || args[0] === "help") {
+        printRaw(HELP);
+        return 0;
+    }
+    const config = readConfig<InferenceConfig>(SERVICE, SHAPE);
+    if (args[0] === "europa-health") return europaHealth(config);
+    if (args[0] === "hermes-releases") return hermesReleases(config, args[1]);
+    const { remote, spec } = buildInferenceRemote(args);
+    const targets: Record<Target, { host: string; user: string; keyFile: string }> = {
+        nexus: { host: config.nexusHost, user: config.nexusUser, keyFile: secretPath(SERVICE, "SSH_KEY_NEXUS") },
+        hermes: { host: config.hermesHost, user: config.hermesUser, keyFile: secretPath(SERVICE, "SSH_KEY_HERMES") },
+        pulsar: { host: config.pulsarHost, user: config.pulsarUser, keyFile: secretPath(SERVICE, "SSH_KEY_PULSAR") },
+    };
+    const result = await sshFixed({ ...targets[spec.target], knownHostsFile: join(specialistsHome(), "config", "known_hosts") }, remote, spec.timeoutMs);
+    if (result.stdout.length > 0) printRaw(result.stdout);
+    if (result.code !== 0) throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
+    return 0;
+}
+
+if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) {
+    await main(SERVICE, command);
+}

package/specialists/install/install.sh

@@ -11,9 +11,9 @@ bin="$root/bin"
 
 [ -d "$src/specialists/lib" ] || { echo "not a pi-astro package: $src" >&2; exit 2; }
 mkdir -p "$bin" "$root/config" "$root/secrets" "$root/work"
-rm -rf "$bin/lib" "$bin/arcane" "$bin/identity" "$bin/network" "$bin/dns" "$bin/edge" "$bin/security"
+rm -rf "$bin/lib" "$bin/arcane" "$bin/identity" "$bin/network" "$bin/dns" "$bin/edge" "$bin/security" "$bin/backup" "$bin/proxmox" "$bin/inference"
 cp -R "$src/specialists/lib" "$bin/lib"
-for s in arcane identity network dns edge security; do
+for s in arcane identity network dns edge security backup proxmox inference; do
   mkdir -p "$bin/$s"
   cp "$src/specialists/$s/run.ts" "$bin/$s/run.ts"
 done

package/specialists/lib/pinned.ts

@@ -0,0 +1,49 @@
+import { request as httpsRequest } from "node:https";
+import { ServiceError } from "./errors.ts";
+
+export interface PinnedResponse {
+    status: number;
+    body: string;
+    fingerprint: string;
+}
+
+export interface PinnedOptions {
+    method: string;
+    headers: Record<string, string>;
+    body?: string;
+    /** SHA-256 fingerprint of the expected certificate (colon-separated hex); null probes without checking. */
+    expectedFingerprint: string | null;
+    timeoutMs: number;
+}
+
+/**
+ * HTTPS request to a self-signed service, accepted only when the peer
+ * certificate matches the pinned fingerprint. Used for consoles that
+ * cannot present a CA-signed certificate (UniFi, Proxmox VE).
+ */
+export function pinnedRequest(url: URL, options: PinnedOptions): Promise<PinnedResponse> {
+    return new Promise((resolve, reject) => {
+        const req = httpsRequest(
+            url,
+            { method: options.method, rejectUnauthorized: false, headers: options.headers, timeout: options.timeoutMs },
+            (res) => {
+                const socket = res.socket as { getPeerCertificate?: () => { fingerprint256?: string } };
+                const fingerprint = socket.getPeerCertificate?.().fingerprint256 ?? "";
+                if (options.expectedFingerprint !== null && fingerprint.toUpperCase() !== options.expectedFingerprint.toUpperCase()) {
+                    res.destroy();
+                    reject(new ServiceError(`${url.host}: certificate fingerprint ${fingerprint} does not match the pinned value`));
+                    return;
+                }
+                let body = "";
+                res.setEncoding("utf-8");
+                res.on("data", (chunk: string) => {
+                    body += chunk;
+                });
+                res.on("end", () => resolve({ status: res.statusCode ?? 0, body, fingerprint }));
+            },
+        );
+        req.on("timeout", () => req.destroy(new ServiceError(`${url.host}: request timed out`)));
+        req.on("error", (err) => reject(err instanceof ServiceError ? err : new ServiceError(`${url.host}: request failed: ${err.message}`)));
+        req.end(options.body);
+    });
+}

package/specialists/network/run.ts

@@ -1,10 +1,10 @@
-import { request as httpsRequest } from "node:https";
 import { join } from "node:path";
 import { readConfig } from "../lib/config.ts";
 import { ServiceError, UsageError } from "../lib/errors.ts";
 import { main } from "../lib/main.ts";
 import { printJson, printRaw } from "../lib/output.ts";
 import { specialistsHome } from "../lib/paths.ts";
+import { pinnedRequest, type PinnedResponse } from "../lib/pinned.ts";
 import { readSecret, secretPath } from "../lib/secrets.ts";
 import { sshFixed } from "../lib/ssh.ts";
 
@@ -114,42 +114,13 @@ export function unifiPath(args: string[]): string {
     return qs.length > 0 ? `${path}?${qs}` : path;
 }
 
-interface PinnedResponse {
-    status: number;
-    body: string;
-    fingerprint: string;
-}
-
 /** GET over TLS pinned to the console certificate; the console is self-signed. */
 export function pinnedGet(url: URL, apiKey: string | null, expectedFingerprint: string | null): Promise<PinnedResponse> {
-    return new Promise((resolve, reject) => {
-        const req = httpsRequest(
-            url,
-            {
-                method: "GET",
-                rejectUnauthorized: false,
-                headers: { Accept: "application/json", ...(apiKey ? { "X-API-KEY": apiKey } : {}) },
-                timeout: 30_000,
-            },
-            (res) => {
-                const socket = res.socket as { getPeerCertificate?: () => { fingerprint256?: string } };
-                const fingerprint = socket.getPeerCertificate?.().fingerprint256 ?? "";
-                if (expectedFingerprint !== null && fingerprint.toUpperCase() !== expectedFingerprint.toUpperCase()) {
-                    res.destroy();
-                    reject(new ServiceError(`console certificate fingerprint ${fingerprint} does not match the pinned value`));
-                    return;
-                }
-                let body = "";
-                res.setEncoding("utf-8");
-                res.on("data", (chunk: string) => {
-                    body += chunk;
-                });
-                res.on("end", () => resolve({ status: res.statusCode ?? 0, body, fingerprint }));
-            },
-        );
-        req.on("timeout", () => req.destroy(new ServiceError("UniFi request timed out")));
-        req.on("error", (err) => reject(err instanceof ServiceError ? err : new ServiceError(`UniFi request failed: ${err.message}`)));
-        req.end();
+    return pinnedRequest(url, {
+        method: "GET",
+        headers: { Accept: "application/json", ...(apiKey ? { "X-API-KEY": apiKey } : {}) },
+        expectedFingerprint,
+        timeoutMs: 30_000,
     });
 }