@astrofoundry/pi-astro 0.40.35 → 0.40.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/daily-run.test.ts +28 -1
- package/extensions/astro-discord/daily-run.ts +65 -4
- package/extensions/astro-discord/index.ts +27 -2
- package/package.json +1 -1
- package/skills/dns/SKILL.md +1 -1
- package/skills/inference/SKILL.md +1 -0
- package/skills/proxmox/SKILL.md +1 -1
- package/skills/security/SKILL.md +4 -2
- package/specialists/dns/run.ts +3 -0
- package/specialists/entry/remote/spc-inference-nexus-entry.sh +1 -0
- package/specialists/inference/run.ts +1 -0
- package/specialists/proxmox/run.ts +1 -1
- package/specialists/wrappers.test.ts +1 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import type { ArcaneReport } from "./daily-report.ts";
|
|
3
3
|
import type { ToolsReport } from "./tools-report.ts";
|
|
4
|
-
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
4
|
+
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
5
5
|
|
|
6
6
|
const EDGE = `nginx active
|
|
7
7
|
wg-quick@wg0 active
|
|
@@ -109,6 +109,33 @@ describe("morning status lines", () => {
|
|
|
109
109
|
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer" });
|
|
110
110
|
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: null, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "no answer from technitium2", problem: "DNS: technitium2 does not answer", route: "admin" });
|
|
111
111
|
expect(dnsStatus({ technitium1: null, technitium2: { status: "error" }, technitium3: { status: "ok" } })).toMatchObject({ problem: "DNS: technitium1 and technitium2 do not answer" });
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it("flags a Mac two macOS minors behind and notes a major without flagging it", () => {
|
|
115
|
+
const nexus = "26.5.2\nTitle: Command Line Tools for Xcode 26.6, Version: 26.6, Size: 920431KiB, Recommended: YES,\nTitle: macOS Tahoe 26.7.2, Version: 26.7.2, Size: 3868257KiB, Recommended: YES, Action: restart,\nTitle: macOS 27.2, Version: 27.2, Size: 11319983KiB, Recommended: YES, Action: restart,";
|
|
116
|
+
const cortex = "26.6.1\nTitle: macOS Tahoe 26.7.2, Version: 26.7.2, Size: 3168375KiB, Recommended: YES, Action: restart,";
|
|
117
|
+
expect(macosStatus([{ name: "Cortex", output: cortex }])).toEqual({ area: "macOS", line: "Cortex 26.6.1, 26.7.2 waiting" });
|
|
118
|
+
expect(macosStatus([{ name: "Nexus", output: nexus }, { name: "Cortex", output: null }])).toEqual({
|
|
119
|
+
area: "macOS",
|
|
120
|
+
line: "Nexus 26.5.2, 26.7.2 waiting, macOS 27.2 offered (major, by hand); Cortex unknown",
|
|
121
|
+
problem: "macOS: Nexus runs macOS 26.5.2 while 26.7.2 is out; automatic installs are not keeping up; no macOS update list from Cortex",
|
|
122
|
+
route: "admin",
|
|
123
|
+
});
|
|
124
|
+
expect(macosStatus([{ name: "Cortex", output: "26.7.2" }]).line).toBe("Cortex 26.7.2");
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("upgrades one Technitium node per morning, Mira first and the primary last, and holds a major", () => {
|
|
128
|
+
const update = (to: string) => ({ status: "ok", response: { updateAvailable: true, updateVersion: to, currentVersion: "15.6.2" } });
|
|
129
|
+
const state = (versions: [string, string, string]) => ({ status: "ok", response: { clusterNodes: versions.map((v, i) => ({ name: `technitium${i + 1}.dns.home.arpa`, version: v })) } });
|
|
130
|
+
expect(technitiumPlan({ status: "ok", response: { updateAvailable: false } }, state(["15.6.2", "15.6.2", "15.6.2"]))).toBeNull();
|
|
131
|
+
expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ host: "technitium3", from: "15.6.2", to: "15.7.0" });
|
|
132
|
+
expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.6.2", "15.7.0"]))).toEqual({ host: "technitium2", from: "15.6.2", to: "15.7.0" });
|
|
133
|
+
expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.7.0", "15.7.0"]))).toEqual({ host: "technitium1", from: "15.6.2", to: "15.7.0" });
|
|
134
|
+
expect(technitiumPlan(update("16.0.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ major: true, from: "15.6.2", to: "16.0.0" });
|
|
135
|
+
const all = { technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } };
|
|
136
|
+
expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: true })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer; technitium3 upgraded 15.6.2 -> 15.7.0 (one node per morning, primary last)" });
|
|
137
|
+
expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: false, why: "installer exited 1" }).problem).toBe("DNS: the upgrade of technitium3 to 15.7.0 failed (installer exited 1)");
|
|
138
|
+
expect(dnsStatus(all, { major: true, from: "15.6.2", to: "16.0.0" }).problem).toMatch(/^DNS: Technitium 16.0.0 is a major release \(running 15.6.2\)/);
|
|
112
139
|
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=200\nrerank \tstate = running \tpid = 81670 health=200")).toEqual({ area: "Inference", line: "Nexus embed, rerank running and healthy" });
|
|
113
140
|
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=503")).toMatchObject({ problem: "Inference: embed not healthy on Nexus" });
|
|
114
141
|
expect(mediaStatus({ sabnzbd_paused: true, warnings: [] })).toEqual({ area: "Media", line: "downloads paused" });
|
|
@@ -187,14 +187,43 @@ export function identityStatus(status: string | null): AreaStatus {
|
|
|
187
187
|
return { area: "Identity", line, ...(bad.length > 0 || units.length === 0 ? { problem: `Identity: ${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM`, route: "admin" as const } : {}) };
|
|
188
188
|
}
|
|
189
189
|
|
|
190
|
-
/**
|
|
191
|
-
|
|
190
|
+
/** Upgrade order: Mira's node first, the primary last, one node per morning. */
|
|
191
|
+
const TECHNITIUM_ORDER = ["technitium3", "technitium2", "technitium1"];
|
|
192
|
+
|
|
193
|
+
/** What the morning run does about a Technitium release: upgrade one node, or hold a major for the owner. */
|
|
194
|
+
export type TechnitiumPlan = { host: string; from: string; to: string } | { major: true; from: string; to: string };
|
|
195
|
+
|
|
196
|
+
/** `check-update` and `cluster-state` from the primary: the next node to upgrade, a held major, or nothing. */
|
|
197
|
+
export function technitiumPlan(update: unknown, state: unknown): TechnitiumPlan | null {
|
|
198
|
+
const u = (update as { response?: { updateAvailable?: unknown; updateVersion?: unknown; currentVersion?: unknown } } | null)?.response;
|
|
199
|
+
if (u?.updateAvailable !== true || typeof u.updateVersion !== "string" || typeof u.currentVersion !== "string") return null;
|
|
200
|
+
const to = u.updateVersion;
|
|
201
|
+
if (to.split(".")[0] !== u.currentVersion.split(".")[0]) return { major: true, from: u.currentVersion, to };
|
|
202
|
+
const nodes = (state as { response?: { clusterNodes?: { name?: unknown; version?: unknown }[] } } | null)?.response?.clusterNodes ?? [];
|
|
203
|
+
for (const host of TECHNITIUM_ORDER) {
|
|
204
|
+
const node = nodes.find((n) => typeof n.name === "string" && n.name.split(".")[0] === host);
|
|
205
|
+
if (node && typeof node.version === "string" && node.version !== to) return { host, from: node.version, to };
|
|
206
|
+
}
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** The morning's Technitium upgrade, if one ran or a major waits. */
|
|
211
|
+
export type DnsUpgrade = { host: string; from: string; to: string; ok: true } | { host: string; from: string; to: string; ok: false; why: string } | { major: true; from: string; to: string };
|
|
212
|
+
|
|
213
|
+
/** `dns technitium primary|node <fqdn> resolve <name>`: every cluster node answers an internal name, keyed by node name; plus the morning's upgrade. */
|
|
214
|
+
export function dnsStatus(answers: Readonly<Record<string, unknown>>, upgrade?: DnsUpgrade): AreaStatus {
|
|
192
215
|
const ok = (r: unknown) => (r as { status?: unknown } | null)?.status === "ok";
|
|
193
216
|
const nodes = Object.keys(answers);
|
|
194
217
|
const down = nodes.filter((node) => !ok(answers[node]));
|
|
195
218
|
const names = down.join(" and ");
|
|
196
|
-
|
|
197
|
-
|
|
219
|
+
const upgradeLine = upgrade === undefined ? null : "major" in upgrade ? `Technitium ${upgrade.to} is a major release and waits for you` : upgrade.ok ? `${upgrade.host} upgraded ${upgrade.from} -> ${upgrade.to} (one node per morning, primary last)` : `the upgrade of ${upgrade.host} to ${upgrade.to} failed`;
|
|
220
|
+
const problems = [
|
|
221
|
+
down.length > 0 ? `${names} ${down.length === 1 ? "does" : "do"} not answer` : null,
|
|
222
|
+
upgrade && "major" in upgrade ? `Technitium ${upgrade.to} is a major release (running ${upgrade.from}); read its change log, then upgrade one node at a time with \`dns technitium host <node> upgrade\`` : null,
|
|
223
|
+
upgrade && "ok" in upgrade && !upgrade.ok ? `the upgrade of ${upgrade.host} to ${upgrade.to} failed (${upgrade.why.slice(0, 200)})` : null,
|
|
224
|
+
].filter(Boolean);
|
|
225
|
+
const line = [down.length === 0 ? `${nodes.join(", ")} answer` : `no answer from ${names}`, upgradeLine].filter(Boolean).join("; ");
|
|
226
|
+
return { area: "DNS", line, ...(problems.length > 0 ? { problem: `DNS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
198
227
|
}
|
|
199
228
|
|
|
200
229
|
/** `inference nexus-status`: each Nexus service running with HTTP health 200. */
|
|
@@ -233,6 +262,38 @@ export function backupStatus(archives: unknown, resticText: string | null, now:
|
|
|
233
262
|
return { area: "Backup", line: `${proxmox}, ${check}`, ...(problems.length > 0 ? { problem: `Backup: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
234
263
|
}
|
|
235
264
|
|
|
265
|
+
/** `sw_vers -productVersion` on its first line, then Software Update's `Title:` lines, for one Mac. */
|
|
266
|
+
export interface MacUpdates {
|
|
267
|
+
name: string;
|
|
268
|
+
output: string | null;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** A Mac this many minor releases behind the newest offered one is not keeping up with its automatic installs. */
|
|
272
|
+
const MACOS_MINORS_BEHIND = 2;
|
|
273
|
+
|
|
274
|
+
/** Each Mac's macOS version and offers: automatic installs cover minors; a major is noted for a manual upgrade. */
|
|
275
|
+
export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
|
|
276
|
+
const parts: string[] = [];
|
|
277
|
+
const problems: string[] = [];
|
|
278
|
+
for (const m of macs) {
|
|
279
|
+
if (m.output === null) {
|
|
280
|
+
parts.push(`${m.name} unknown`);
|
|
281
|
+
problems.push(`no macOS update list from ${m.name}`);
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
const [installed = "", ...rest] = m.output.split("\n").map((l) => l.trim());
|
|
285
|
+
const offers = rest.map((l) => /Title: macOS[^,]*, Version: ([\d.]+)/.exec(l)?.[1]).filter((v): v is string => v !== undefined);
|
|
286
|
+
const [major, minor] = installed.split(".").map(Number);
|
|
287
|
+
const sameMajor = offers.filter((v) => Number(v.split(".")[0]) === major);
|
|
288
|
+
const newestMinor = Math.max(minor ?? 0, ...sameMajor.map((v) => Number(v.split(".")[1] ?? 0)));
|
|
289
|
+
const majors = offers.filter((v) => Number(v.split(".")[0]) > major);
|
|
290
|
+
const latest = sameMajor.sort((a, b) => a.localeCompare(b, undefined, { numeric: true })).at(-1);
|
|
291
|
+
parts.push(`${m.name} ${installed}${latest ? `, ${latest} waiting` : ""}${majors.length > 0 ? `, macOS ${majors[0]} offered (major, by hand)` : ""}`);
|
|
292
|
+
if (newestMinor - (minor ?? 0) >= MACOS_MINORS_BEHIND) problems.push(`${m.name} runs macOS ${installed} while ${latest} is out; automatic installs are not keeping up`);
|
|
293
|
+
}
|
|
294
|
+
return { area: "macOS", line: parts.join("; "), ...(problems.length > 0 ? { problem: `macOS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
295
|
+
}
|
|
296
|
+
|
|
236
297
|
/** One report the run posted in its own channel, as the summary shows it. */
|
|
237
298
|
export interface ReportLine {
|
|
238
299
|
area: string;
|
|
@@ -78,7 +78,7 @@ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
|
|
|
78
78
|
import { parseSnapshot } from "./arcane-watch.ts";
|
|
79
79
|
import { type Applied, autoFailureText, foremanTask, stackProblems, stackUpdateAnswer, SUPERVISE_DELAY_MS, SUPERVISE_LIMIT_MS } from "./supervise.ts";
|
|
80
80
|
import { stackPlan, toolPlan } from "./update-policy.ts";
|
|
81
|
-
import { type AreaStatus, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
81
|
+
import { type AreaStatus, arcaneSummary, backupStatus, type DnsUpgrade, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
82
82
|
import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
|
|
83
83
|
import { nasOutage, readSecurityState } from "./security.ts";
|
|
84
84
|
|
|
@@ -179,6 +179,8 @@ const PROXMOX_HOSTS: readonly { name: string; id: string; prefix: string[] }[] =
|
|
|
179
179
|
{ name: "Pulsar", id: "pulsar", prefix: [] },
|
|
180
180
|
{ name: "SPACE-X", id: "space-x", prefix: ["space-x"] },
|
|
181
181
|
];
|
|
182
|
+
/** The dns wrapper allows a node upgrade 10 minutes; the bridge waits a little longer. */
|
|
183
|
+
const DNS_UPGRADE_MS = 11 * 60_000;
|
|
182
184
|
/** The wrapper allows a host upgrade 20 minutes; the bridge waits a little longer for its answer. */
|
|
183
185
|
const HOST_UPGRADE_MS = 21 * 60_000;
|
|
184
186
|
|
|
@@ -671,17 +673,40 @@ class Bridge {
|
|
|
671
673
|
if (present.has("dns")) {
|
|
672
674
|
areas.push(
|
|
673
675
|
(async () => {
|
|
676
|
+
// A minor Technitium release upgrades one node per morning, before the answer checks below.
|
|
677
|
+
const plan = technitiumPlan(await json("dns", ["technitium", "primary", "check-update"]), await json("dns", ["technitium", "primary", "cluster-state"]));
|
|
678
|
+
let upgrade: DnsUpgrade | undefined;
|
|
679
|
+
if (plan && "major" in plan) upgrade = plan;
|
|
680
|
+
else if (plan) {
|
|
681
|
+
upgrade = await this.runEntry("dns", AUTO_CALLER, ["technitium", "host", plan.host, "upgrade"], DNS_UPGRADE_MS).then(
|
|
682
|
+
() => ({ ...plan, ok: true as const }),
|
|
683
|
+
(err) => ({ ...plan, ok: false as const, why: err instanceof Error ? err.message : String(err) }),
|
|
684
|
+
);
|
|
685
|
+
}
|
|
674
686
|
const [technitium1, technitium2, technitium3] = await Promise.all([
|
|
675
687
|
json("dns", ["technitium", "primary", "resolve", DNS_PROBE]),
|
|
676
688
|
json("dns", ["technitium", "node", DNS_NODES.technitium2, "resolve", DNS_PROBE]),
|
|
677
689
|
json("dns", ["technitium", "node", DNS_NODES.technitium3, "resolve", DNS_PROBE]),
|
|
678
690
|
]);
|
|
679
|
-
return dnsStatus({ technitium1, technitium2, technitium3 });
|
|
691
|
+
return dnsStatus({ technitium1, technitium2, technitium3 }, upgrade);
|
|
680
692
|
})(),
|
|
681
693
|
);
|
|
682
694
|
}
|
|
683
695
|
if (present.has("inference")) areas.push((async () => inferenceStatus(await text("inference", ["nexus-status"])))());
|
|
684
696
|
if (present.has(MEDIA)) areas.push((async () => mediaStatus(await json(MEDIA, ["sabnzbd", "status"])))());
|
|
697
|
+
areas.push(
|
|
698
|
+
(async () => {
|
|
699
|
+
// Cortex is this host; Nexus answers through the inference specialist.
|
|
700
|
+
const version = await this.execTool("/usr/bin/sw_vers", ["-productVersion"], { timeoutMs: 30_000, env: { PATH: TOOL_PATH } });
|
|
701
|
+
const offers = await this.execTool("/usr/sbin/softwareupdate", ["-l"], { timeoutMs: 3 * 60_000, env: { PATH: TOOL_PATH } });
|
|
702
|
+
const cortex = version.code === 0 ? `${version.stdout.trim()}\n${offers.stdout}${offers.stderr}` : null;
|
|
703
|
+
const nexus = present.has("inference") ? await text("inference", ["macos-updates"]) : null;
|
|
704
|
+
return macosStatus([
|
|
705
|
+
{ name: "Cortex", output: cortex },
|
|
706
|
+
...(present.has("inference") ? [{ name: "Nexus", output: nexus }] : []),
|
|
707
|
+
]);
|
|
708
|
+
})(),
|
|
709
|
+
);
|
|
685
710
|
return Promise.all(areas);
|
|
686
711
|
}
|
|
687
712
|
|
package/package.json
CHANGED
package/skills/dns/SKILL.md
CHANGED
|
@@ -66,7 +66,7 @@ Fixed operations on each node's host through one forced-command SSH key (`spc-dn
|
|
|
66
66
|
| `restart` | restart Technitium and print its state |
|
|
67
67
|
| `upgrade` | run the official installer (`download.technitium.com/dns/install.sh`), which upgrades Technitium and its ASP.NET runtime in place; up to 10 minutes |
|
|
68
68
|
|
|
69
|
-
Upgrade or restart one node at a time and check `cluster-state` and `resolve` on it before the next; the other two nodes keep answering. The wrapper holds one lock for `restart` and `upgrade` across all nodes and refuses a second one while it runs. Upgrade the secondaries first and technitium1 last, and keep all three on the same version.
|
|
69
|
+
Upgrade or restart one node at a time and check `cluster-state` and `resolve` on it before the next; the other two nodes keep answering. The morning run does this on its own for a minor release (`technitium primary check-update`): technitium3, then technitium2, then the primary, one node per morning; a new major waits for the owner. The wrapper holds one lock for `restart` and `upgrade` across all nodes and refuses a second one while it runs. Upgrade the secondaries first and technitium1 last, and keep all three on the same version.
|
|
70
70
|
|
|
71
71
|
```json
|
|
72
72
|
{ "args": ["technitium", "host", "technitium3", "status"] }
|
|
@@ -17,6 +17,7 @@ Two hosts serve AI workloads. Nexus (Mac Studio, `10.0.40.38`, user `nexus`) run
|
|
|
17
17
|
| `nexus-stop <embed|rerank> --confirm` | unload the service |
|
|
18
18
|
| `nexus-log <embed|rerank> <lines>` | tail of `~/llama-<service>.log` |
|
|
19
19
|
| `nexus-disk` | free space |
|
|
20
|
+
| `macos-updates` | installed macOS version and Software Update's offers (Nexus installs minor updates on its own; the morning run flags it two minors behind) |
|
|
20
21
|
|
|
21
22
|
A model load after restart takes up to a minute; `health=503` during that time is loading, not failure.
|
|
22
23
|
|
package/skills/proxmox/SKILL.md
CHANGED
|
@@ -5,7 +5,7 @@ description: How the proxmox specialist operates Proxmox VE on Pulsar (37) and S
|
|
|
5
5
|
|
|
6
6
|
# Proxmox specialist
|
|
7
7
|
|
|
8
|
-
Pulsar (`10.0.10.37`, Proxmox VE 9 on Debian 13, node name `pulsar`) runs every guest: VM 100 `arcane` (Docker platform), LXC 101 `hermes` (
|
|
8
|
+
Pulsar (`10.0.10.37`, Proxmox VE 9 on Debian 13, node name `pulsar`) runs every guest: VM 100 `arcane` (Docker platform), LXC 101 `hermes` (stopped and retired on 2026-11-01; the operator deletes it after 2026-12-01; never start it), LXC 102 `backup` (GCS pusher), VM 103 `identity` (Zitadel), VM 104 `dmz` (Pomerium), LXC 105 `tailscale`, VM 106 `observability`, LXC 107 `technitium2`, LXC 108 `radius`. Storage `local-lvm` holds the disks, `unas-backup` the vzdump archives. The `proxmox` tool talks to the API with a token whose role allows audit everywhere and, on guests, power, snapshots, and CPU/memory/option/network settings; nothing else. `["--help"]` lists the commands.
|
|
9
9
|
|
|
10
10
|
SPACE-X (`10.1.10.37`, Proxmox VE 9 at Mira, node name `space-x`) runs LXC 100 `technitium3` (DNS cluster node) and LXC 105 `tailscale` (Mira's subnet router). Put `space-x` before any command to run it there, for example `["space-x", "guests"]`; without it every command runs on Pulsar. SPACE-X has the same restricted user `spc-proxmox@pve!cortex` with its own token, and the same forced-command key as Pulsar. SPACE-X backs up every guest nightly at 02:30 (Dubai time) to storage `tesla-backup` on Mira's Synology TESLA; Pulsar's `mira-backup-pull` copies the archives to 37 at 05:00. `["space-x", "tasks"]` shows the vzdump runs.
|
|
11
11
|
|
package/skills/security/SKILL.md
CHANGED
|
@@ -5,7 +5,7 @@ description: How the security specialist reads Wazuh, CrowdSec, and the remote l
|
|
|
5
5
|
|
|
6
6
|
# Security specialist
|
|
7
7
|
|
|
8
|
-
VM 106 (`10.0.90.10`, VLAN 90) runs the Wazuh manager (container `observability-wazuh-manager-1`, agents `identity`, `arcane`, `dmz`, `CORTEX`, `NEXUS`, `
|
|
8
|
+
VM 106 (`10.0.90.10`, VLAN 90) runs the Wazuh manager (container `observability-wazuh-manager-1`, agents `identity`, `arcane`, `dmz`, `CORTEX`, `NEXUS`, `frontdoor-1337`, `tailscale`, and since 2026-11-01 `pulsar`, `synapse`, `technitium2`, `radius`, `backup`, `space-x`, `technitium3`, `tailscale-mira`), CrowdSec with its local API for the front-door bouncer, and rsyslog, which archives remote logs under `/var/log/remote/<host>/<program>.log` (14 days). The Discord bridge on Cortex reads Wazuh through this tool every two minutes and posts urgent alerts at once (level 11 or higher, and rules 100011, 100012, 100020, 100021; a deployment-file change that a GitOps commit or our own operation explains is not urgent), with Investigate and Acknowledge buttons for the owners; everything else goes into the Security Daily Report (06:00) and, on Mondays, the Security Weekly Report. The VM itself no longer posts to Discord. Two local CrowdSec scenarios matter: `37pla/pomerium-http-probing` (repeated Pomerium denials) and `37pla/frontdoor-sni-scanning` (rejected SNI at the VPS; bans escalate 4h, 24h, 7d, 30d, 90d). The home WAN address is whitelisted every five minutes; a scan test in August 2026 once banned it.
|
|
9
9
|
|
|
10
10
|
The `security` tool is read-only. Every call is a fixed command on the guest; `["--help"]` lists them.
|
|
11
11
|
|
|
@@ -54,7 +54,9 @@ The `security` tool is read-only. Every call is a fixed command on the guest; `[
|
|
|
54
54
|
| `arcane` | VM 100, Docker host | `astro.arcane` for containers |
|
|
55
55
|
| `identity` | VM 103, ZITADEL | `astro.identity` |
|
|
56
56
|
| `dmz` | VM 104, Pomerium and nginx | `astro.identity` |
|
|
57
|
-
| `
|
|
57
|
+
| `pulsar`, `space-x` | Proxmox hosts at 37 and Mira | `astro.proxmox` |
|
|
58
|
+
| `synapse`, `technitium2`, `technitium3` | the three DNS nodes | `astro.dns` (`technitium host <node>`) |
|
|
59
|
+
| `radius`, `backup`, `tailscale-mira` | LXC 108, LXC 102, SPACE-X LXC 105 | `astro.network` (RADIUS), `astro.backup`, none for the Mira router |
|
|
58
60
|
| `NEXUS` | Mac Studio | `astro.inference` |
|
|
59
61
|
| `CORTEX`, `tailscale` | Mac mini, LXC 105 | none; the operator |
|
|
60
62
|
|
package/specialists/dns/run.ts
CHANGED
|
@@ -106,6 +106,7 @@ export const TECHNITIUM_READS: Readonly<Record<string, { args: [number, number];
|
|
|
106
106
|
stats: { args: [0, 1], help: `stats [${STATS_TYPES.join("|")}] dashboard counters` },
|
|
107
107
|
"cluster-state": { args: [0, 0], help: "cluster-state cluster nodes, their state, and this server's addresses" },
|
|
108
108
|
settings: { args: [0, 0], help: "settings all server settings (secrets redacted)" },
|
|
109
|
+
"check-update": { args: [0, 0], help: "check-update (primary) whether a newer Technitium version exists" },
|
|
109
110
|
};
|
|
110
111
|
|
|
111
112
|
/** Technitium writes; zone and cluster writes on the primary only, the other nodes follow through the catalog and the cluster. */
|
|
@@ -224,6 +225,8 @@ export function technitiumRequest(catalog: string, args: string[]): TechnitiumRe
|
|
|
224
225
|
return { path: "/api/admin/cluster/state", params: { includeServerIpAddresses: "true" }, primaryOnly: false };
|
|
225
226
|
case "settings":
|
|
226
227
|
return { path: "/api/settings/get", params: {}, primaryOnly: false };
|
|
228
|
+
case "check-update":
|
|
229
|
+
return { path: "/api/user/checkForUpdate", params: {}, primaryOnly: true };
|
|
227
230
|
case "cluster-init": {
|
|
228
231
|
if (!IP_LIST.test(rest[1])) throw new UsageError("cluster-init needs a comma separated list of IP addresses");
|
|
229
232
|
return {
|
|
@@ -35,5 +35,6 @@ case "$name" in
|
|
|
35
35
|
printf '%s' "$lines" | grep -Eq '^[1-9][0-9]{0,3}$' || { echo "bad lines" >&2; exit 2; }
|
|
36
36
|
tail -n "$lines" "$HOME/llama-$svc.log" ;;
|
|
37
37
|
nexus-disk) df -h / ;;
|
|
38
|
+
macos-updates) [ -z "$rest" ] || { echo "macos-updates takes no arguments" >&2; exit 2; }; sw_vers -productVersion; softwareupdate -l 2>&1 | grep -E 'Title:' || true ;;
|
|
38
39
|
*) echo "refused: $cmd" >&2; exit 2 ;;
|
|
39
40
|
esac
|
|
@@ -39,6 +39,7 @@ export const INFERENCE_COMMANDS: Readonly<Record<string, RemoteSpec>> = {
|
|
|
39
39
|
"nexus-stop": { args: 1, help: "nexus-stop <embed|rerank> --confirm launchctl bootout", timeoutMs: 2 * MINUTE, confirm: true },
|
|
40
40
|
"nexus-log": { args: 2, help: "nexus-log <embed|rerank> <lines>", timeoutMs: MINUTE },
|
|
41
41
|
"nexus-disk": { args: 0, help: "free space on the Nexus disk", timeoutMs: MINUTE },
|
|
42
|
+
"macos-updates": { args: 0, help: "installed macOS version and the updates Software Update offers", timeoutMs: 3 * MINUTE },
|
|
42
43
|
};
|
|
43
44
|
|
|
44
45
|
const NEXUS_SERVICES = ["embed", "rerank"];
|
|
@@ -113,7 +113,7 @@ ${Object.entries(HOST_COMMANDS)
|
|
|
113
113
|
.map(([name, c]) => ` ${name.padEnd(35)} ${c.help}`)
|
|
114
114
|
.join("\n")}
|
|
115
115
|
|
|
116
|
-
Not available: create, clone, destroy, disk changes, host
|
|
116
|
+
Not available: create, clone, destroy, disk changes, host configuration. Those are operator tasks.`;
|
|
117
117
|
|
|
118
118
|
interface PveEnvelope {
|
|
119
119
|
data?: unknown;
|
|
@@ -216,6 +216,7 @@ describe("network wrapper", () => {
|
|
|
216
216
|
describe("dns wrapper", () => {
|
|
217
217
|
it("maps technitium commands to documented API paths", () => {
|
|
218
218
|
expect(technitiumRequest("catalog.invalid", ["zones"])).toEqual({ path: "/api/zones/list", params: {}, primaryOnly: false });
|
|
219
|
+
expect(technitiumRequest("catalog.invalid", ["check-update"])).toEqual({ path: "/api/user/checkForUpdate", params: {}, primaryOnly: true });
|
|
219
220
|
expect(technitiumRequest("catalog.invalid", ["zones", "*.37pla.net"])).toEqual({ path: "/api/zones/list", params: { filterName: "*.37pla.net" }, primaryOnly: false });
|
|
220
221
|
expect(technitiumRequest("catalog.invalid", ["records", "37pla.net"])).toEqual({
|
|
221
222
|
path: "/api/zones/records/get",
|