@astrofoundry/pi-astro 0.40.35 → 0.40.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import { describe, expect, it } from "vitest";
2
2
  import type { ArcaneReport } from "./daily-report.ts";
3
3
  import type { ToolsReport } from "./tools-report.ts";
4
- import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
4
+ import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
5
5
 
6
6
  const EDGE = `nginx active
7
7
  wg-quick@wg0 active
@@ -109,6 +109,33 @@ describe("morning status lines", () => {
109
109
  expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer" });
110
110
  expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: null, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "no answer from technitium2", problem: "DNS: technitium2 does not answer", route: "admin" });
111
111
  expect(dnsStatus({ technitium1: null, technitium2: { status: "error" }, technitium3: { status: "ok" } })).toMatchObject({ problem: "DNS: technitium1 and technitium2 do not answer" });
112
+ });
113
+
114
+ it("flags a Mac two macOS minors behind and notes a major without flagging it", () => {
115
+ const nexus = "26.5.2\nTitle: Command Line Tools for Xcode 26.6, Version: 26.6, Size: 920431KiB, Recommended: YES,\nTitle: macOS Tahoe 26.7.2, Version: 26.7.2, Size: 3868257KiB, Recommended: YES, Action: restart,\nTitle: macOS 27.2, Version: 27.2, Size: 11319983KiB, Recommended: YES, Action: restart,";
116
+ const cortex = "26.6.1\nTitle: macOS Tahoe 26.7.2, Version: 26.7.2, Size: 3168375KiB, Recommended: YES, Action: restart,";
117
+ expect(macosStatus([{ name: "Cortex", output: cortex }])).toEqual({ area: "macOS", line: "Cortex 26.6.1, 26.7.2 waiting" });
118
+ expect(macosStatus([{ name: "Nexus", output: nexus }, { name: "Cortex", output: null }])).toEqual({
119
+ area: "macOS",
120
+ line: "Nexus 26.5.2, 26.7.2 waiting, macOS 27.2 offered (major, by hand); Cortex unknown",
121
+ problem: "macOS: Nexus runs macOS 26.5.2 while 26.7.2 is out; automatic installs are not keeping up; no macOS update list from Cortex",
122
+ route: "admin",
123
+ });
124
+ expect(macosStatus([{ name: "Cortex", output: "26.7.2" }]).line).toBe("Cortex 26.7.2");
125
+ });
126
+
127
+ it("upgrades one Technitium node per morning, Mira first and the primary last, and holds a major", () => {
128
+ const update = (to: string) => ({ status: "ok", response: { updateAvailable: true, updateVersion: to, currentVersion: "15.6.2" } });
129
+ const state = (versions: [string, string, string]) => ({ status: "ok", response: { clusterNodes: versions.map((v, i) => ({ name: `technitium${i + 1}.dns.home.arpa`, version: v })) } });
130
+ expect(technitiumPlan({ status: "ok", response: { updateAvailable: false } }, state(["15.6.2", "15.6.2", "15.6.2"]))).toBeNull();
131
+ expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ host: "technitium3", from: "15.6.2", to: "15.7.0" });
132
+ expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.6.2", "15.7.0"]))).toEqual({ host: "technitium2", from: "15.6.2", to: "15.7.0" });
133
+ expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.7.0", "15.7.0"]))).toEqual({ host: "technitium1", from: "15.6.2", to: "15.7.0" });
134
+ expect(technitiumPlan(update("16.0.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ major: true, from: "15.6.2", to: "16.0.0" });
135
+ const all = { technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } };
136
+ expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: true })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer; technitium3 upgraded 15.6.2 -> 15.7.0 (one node per morning, primary last)" });
137
+ expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: false, why: "installer exited 1" }).problem).toBe("DNS: the upgrade of technitium3 to 15.7.0 failed (installer exited 1)");
138
+ expect(dnsStatus(all, { major: true, from: "15.6.2", to: "16.0.0" }).problem).toMatch(/^DNS: Technitium 16.0.0 is a major release \(running 15.6.2\)/);
112
139
  expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=200\nrerank \tstate = running \tpid = 81670 health=200")).toEqual({ area: "Inference", line: "Nexus embed, rerank running and healthy" });
113
140
  expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=503")).toMatchObject({ problem: "Inference: embed not healthy on Nexus" });
114
141
  expect(mediaStatus({ sabnzbd_paused: true, warnings: [] })).toEqual({ area: "Media", line: "downloads paused" });
@@ -187,14 +187,43 @@ export function identityStatus(status: string | null): AreaStatus {
187
187
  return { area: "Identity", line, ...(bad.length > 0 || units.length === 0 ? { problem: `Identity: ${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM`, route: "admin" as const } : {}) };
188
188
  }
189
189
 
190
- /** `dns technitium primary|node <fqdn> resolve <name>`: every cluster node answers an internal name, keyed by node name. */
191
- export function dnsStatus(answers: Readonly<Record<string, unknown>>): AreaStatus {
190
+ /** Upgrade order: Mira's node first, the primary last, one node per morning. */
191
+ const TECHNITIUM_ORDER = ["technitium3", "technitium2", "technitium1"];
192
+
193
+ /** What the morning run does about a Technitium release: upgrade one node, or hold a major for the owner. */
194
+ export type TechnitiumPlan = { host: string; from: string; to: string } | { major: true; from: string; to: string };
195
+
196
+ /** `check-update` and `cluster-state` from the primary: the next node to upgrade, a held major, or nothing. */
197
+ export function technitiumPlan(update: unknown, state: unknown): TechnitiumPlan | null {
198
+ const u = (update as { response?: { updateAvailable?: unknown; updateVersion?: unknown; currentVersion?: unknown } } | null)?.response;
199
+ if (u?.updateAvailable !== true || typeof u.updateVersion !== "string" || typeof u.currentVersion !== "string") return null;
200
+ const to = u.updateVersion;
201
+ if (to.split(".")[0] !== u.currentVersion.split(".")[0]) return { major: true, from: u.currentVersion, to };
202
+ const nodes = (state as { response?: { clusterNodes?: { name?: unknown; version?: unknown }[] } } | null)?.response?.clusterNodes ?? [];
203
+ for (const host of TECHNITIUM_ORDER) {
204
+ const node = nodes.find((n) => typeof n.name === "string" && n.name.split(".")[0] === host);
205
+ if (node && typeof node.version === "string" && node.version !== to) return { host, from: node.version, to };
206
+ }
207
+ return null;
208
+ }
209
+
210
+ /** The morning's Technitium upgrade, if one ran or a major waits. */
211
+ export type DnsUpgrade = { host: string; from: string; to: string; ok: true } | { host: string; from: string; to: string; ok: false; why: string } | { major: true; from: string; to: string };
212
+
213
+ /** `dns technitium primary|node <fqdn> resolve <name>`: every cluster node answers an internal name, keyed by node name; plus the morning's upgrade. */
214
+ export function dnsStatus(answers: Readonly<Record<string, unknown>>, upgrade?: DnsUpgrade): AreaStatus {
192
215
  const ok = (r: unknown) => (r as { status?: unknown } | null)?.status === "ok";
193
216
  const nodes = Object.keys(answers);
194
217
  const down = nodes.filter((node) => !ok(answers[node]));
195
218
  const names = down.join(" and ");
196
- if (down.length === 0) return { area: "DNS", line: `${nodes.join(", ")} answer` };
197
- return { area: "DNS", line: `no answer from ${names}`, problem: `DNS: ${names} ${down.length === 1 ? "does" : "do"} not answer`, route: "admin" };
219
+ const upgradeLine = upgrade === undefined ? null : "major" in upgrade ? `Technitium ${upgrade.to} is a major release and waits for you` : upgrade.ok ? `${upgrade.host} upgraded ${upgrade.from} -> ${upgrade.to} (one node per morning, primary last)` : `the upgrade of ${upgrade.host} to ${upgrade.to} failed`;
220
+ const problems = [
221
+ down.length > 0 ? `${names} ${down.length === 1 ? "does" : "do"} not answer` : null,
222
+ upgrade && "major" in upgrade ? `Technitium ${upgrade.to} is a major release (running ${upgrade.from}); read its change log, then upgrade one node at a time with \`dns technitium host <node> upgrade\`` : null,
223
+ upgrade && "ok" in upgrade && !upgrade.ok ? `the upgrade of ${upgrade.host} to ${upgrade.to} failed (${upgrade.why.slice(0, 200)})` : null,
224
+ ].filter(Boolean);
225
+ const line = [down.length === 0 ? `${nodes.join(", ")} answer` : `no answer from ${names}`, upgradeLine].filter(Boolean).join("; ");
226
+ return { area: "DNS", line, ...(problems.length > 0 ? { problem: `DNS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
198
227
  }
199
228
 
200
229
  /** `inference nexus-status`: each Nexus service running with HTTP health 200. */
@@ -233,6 +262,38 @@ export function backupStatus(archives: unknown, resticText: string | null, now:
233
262
  return { area: "Backup", line: `${proxmox}, ${check}`, ...(problems.length > 0 ? { problem: `Backup: ${problems.join("; ")}`, route: "admin" as const } : {}) };
234
263
  }
235
264
 
265
+ /** `sw_vers -productVersion` on its first line, then Software Update's `Title:` lines, for one Mac. */
266
+ export interface MacUpdates {
267
+ name: string;
268
+ output: string | null;
269
+ }
270
+
271
+ /** A Mac this many minor releases behind the newest offered one is not keeping up with its automatic installs. */
272
+ const MACOS_MINORS_BEHIND = 2;
273
+
274
+ /** Each Mac's macOS version and offers: automatic installs cover minors; a major is noted for a manual upgrade. */
275
+ export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
276
+ const parts: string[] = [];
277
+ const problems: string[] = [];
278
+ for (const m of macs) {
279
+ if (m.output === null) {
280
+ parts.push(`${m.name} unknown`);
281
+ problems.push(`no macOS update list from ${m.name}`);
282
+ continue;
283
+ }
284
+ const [installed = "", ...rest] = m.output.split("\n").map((l) => l.trim());
285
+ const offers = rest.map((l) => /Title: macOS[^,]*, Version: ([\d.]+)/.exec(l)?.[1]).filter((v): v is string => v !== undefined);
286
+ const [major, minor] = installed.split(".").map(Number);
287
+ const sameMajor = offers.filter((v) => Number(v.split(".")[0]) === major);
288
+ const newestMinor = Math.max(minor ?? 0, ...sameMajor.map((v) => Number(v.split(".")[1] ?? 0)));
289
+ const majors = offers.filter((v) => Number(v.split(".")[0]) > major);
290
+ const latest = sameMajor.sort((a, b) => a.localeCompare(b, undefined, { numeric: true })).at(-1);
291
+ parts.push(`${m.name} ${installed}${latest ? `, ${latest} waiting` : ""}${majors.length > 0 ? `, macOS ${majors[0]} offered (major, by hand)` : ""}`);
292
+ if (newestMinor - (minor ?? 0) >= MACOS_MINORS_BEHIND) problems.push(`${m.name} runs macOS ${installed} while ${latest} is out; automatic installs are not keeping up`);
293
+ }
294
+ return { area: "macOS", line: parts.join("; "), ...(problems.length > 0 ? { problem: `macOS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
295
+ }
296
+
236
297
  /** One report the run posted in its own channel, as the summary shows it. */
237
298
  export interface ReportLine {
238
299
  area: string;
@@ -78,7 +78,7 @@ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
78
78
  import { parseSnapshot } from "./arcane-watch.ts";
79
79
  import { type Applied, autoFailureText, foremanTask, stackProblems, stackUpdateAnswer, SUPERVISE_DELAY_MS, SUPERVISE_LIMIT_MS } from "./supervise.ts";
80
80
  import { stackPlan, toolPlan } from "./update-policy.ts";
81
- import { type AreaStatus, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
81
+ import { type AreaStatus, arcaneSummary, backupStatus, type DnsUpgrade, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
82
82
  import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
83
83
  import { nasOutage, readSecurityState } from "./security.ts";
84
84
 
@@ -179,6 +179,8 @@ const PROXMOX_HOSTS: readonly { name: string; id: string; prefix: string[] }[] =
179
179
  { name: "Pulsar", id: "pulsar", prefix: [] },
180
180
  { name: "SPACE-X", id: "space-x", prefix: ["space-x"] },
181
181
  ];
182
+ /** The dns wrapper allows a node upgrade 10 minutes; the bridge waits a little longer. */
183
+ const DNS_UPGRADE_MS = 11 * 60_000;
182
184
  /** The wrapper allows a host upgrade 20 minutes; the bridge waits a little longer for its answer. */
183
185
  const HOST_UPGRADE_MS = 21 * 60_000;
184
186
 
@@ -671,17 +673,40 @@ class Bridge {
671
673
  if (present.has("dns")) {
672
674
  areas.push(
673
675
  (async () => {
676
+ // A minor Technitium release upgrades one node per morning, before the answer checks below.
677
+ const plan = technitiumPlan(await json("dns", ["technitium", "primary", "check-update"]), await json("dns", ["technitium", "primary", "cluster-state"]));
678
+ let upgrade: DnsUpgrade | undefined;
679
+ if (plan && "major" in plan) upgrade = plan;
680
+ else if (plan) {
681
+ upgrade = await this.runEntry("dns", AUTO_CALLER, ["technitium", "host", plan.host, "upgrade"], DNS_UPGRADE_MS).then(
682
+ () => ({ ...plan, ok: true as const }),
683
+ (err) => ({ ...plan, ok: false as const, why: err instanceof Error ? err.message : String(err) }),
684
+ );
685
+ }
674
686
  const [technitium1, technitium2, technitium3] = await Promise.all([
675
687
  json("dns", ["technitium", "primary", "resolve", DNS_PROBE]),
676
688
  json("dns", ["technitium", "node", DNS_NODES.technitium2, "resolve", DNS_PROBE]),
677
689
  json("dns", ["technitium", "node", DNS_NODES.technitium3, "resolve", DNS_PROBE]),
678
690
  ]);
679
- return dnsStatus({ technitium1, technitium2, technitium3 });
691
+ return dnsStatus({ technitium1, technitium2, technitium3 }, upgrade);
680
692
  })(),
681
693
  );
682
694
  }
683
695
  if (present.has("inference")) areas.push((async () => inferenceStatus(await text("inference", ["nexus-status"])))());
684
696
  if (present.has(MEDIA)) areas.push((async () => mediaStatus(await json(MEDIA, ["sabnzbd", "status"])))());
697
+ areas.push(
698
+ (async () => {
699
+ // Cortex is this host; Nexus answers through the inference specialist.
700
+ const version = await this.execTool("/usr/bin/sw_vers", ["-productVersion"], { timeoutMs: 30_000, env: { PATH: TOOL_PATH } });
701
+ const offers = await this.execTool("/usr/sbin/softwareupdate", ["-l"], { timeoutMs: 3 * 60_000, env: { PATH: TOOL_PATH } });
702
+ const cortex = version.code === 0 ? `${version.stdout.trim()}\n${offers.stdout}${offers.stderr}` : null;
703
+ const nexus = present.has("inference") ? await text("inference", ["macos-updates"]) : null;
704
+ return macosStatus([
705
+ { name: "Cortex", output: cortex },
706
+ ...(present.has("inference") ? [{ name: "Nexus", output: nexus }] : []),
707
+ ]);
708
+ })(),
709
+ );
685
710
  return Promise.all(areas);
686
711
  }
687
712
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.40.35",
3
+ "version": "0.40.36",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -66,7 +66,7 @@ Fixed operations on each node's host through one forced-command SSH key (`spc-dn
66
66
  | `restart` | restart Technitium and print its state |
67
67
  | `upgrade` | run the official installer (`download.technitium.com/dns/install.sh`), which upgrades Technitium and its ASP.NET runtime in place; up to 10 minutes |
68
68
 
69
- Upgrade or restart one node at a time and check `cluster-state` and `resolve` on it before the next; the other two nodes keep answering. The wrapper holds one lock for `restart` and `upgrade` across all nodes and refuses a second one while it runs. Upgrade the secondaries first and technitium1 last, and keep all three on the same version.
69
+ Upgrade or restart one node at a time and check `cluster-state` and `resolve` on it before the next; the other two nodes keep answering. The morning run does this on its own for a minor release (`technitium primary check-update`): technitium3, then technitium2, then the primary, one node per morning; a new major waits for the owner. The wrapper holds one lock for `restart` and `upgrade` across all nodes and refuses a second one while it runs. Upgrade the secondaries first and technitium1 last, and keep all three on the same version.
70
70
 
71
71
  ```json
72
72
  { "args": ["technitium", "host", "technitium3", "status"] }
@@ -17,6 +17,7 @@ Two hosts serve AI workloads. Nexus (Mac Studio, `10.0.40.38`, user `nexus`) run
17
17
  | `nexus-stop <embed|rerank> --confirm` | unload the service |
18
18
  | `nexus-log <embed|rerank> <lines>` | tail of `~/llama-<service>.log` |
19
19
  | `nexus-disk` | free space |
20
+ | `macos-updates` | installed macOS version and Software Update's offers (Nexus installs minor updates on its own; the morning run flags it two minors behind) |
20
21
 
21
22
  A model load after restart takes up to a minute; `health=503` during that time is loading, not failure.
22
23
 
@@ -106,6 +106,7 @@ export const TECHNITIUM_READS: Readonly<Record<string, { args: [number, number];
106
106
  stats: { args: [0, 1], help: `stats [${STATS_TYPES.join("|")}] dashboard counters` },
107
107
  "cluster-state": { args: [0, 0], help: "cluster-state cluster nodes, their state, and this server's addresses" },
108
108
  settings: { args: [0, 0], help: "settings all server settings (secrets redacted)" },
109
+ "check-update": { args: [0, 0], help: "check-update (primary) whether a newer Technitium version exists" },
109
110
  };
110
111
 
111
112
  /** Technitium writes; zone and cluster writes on the primary only, the other nodes follow through the catalog and the cluster. */
@@ -224,6 +225,8 @@ export function technitiumRequest(catalog: string, args: string[]): TechnitiumRe
224
225
  return { path: "/api/admin/cluster/state", params: { includeServerIpAddresses: "true" }, primaryOnly: false };
225
226
  case "settings":
226
227
  return { path: "/api/settings/get", params: {}, primaryOnly: false };
228
+ case "check-update":
229
+ return { path: "/api/user/checkForUpdate", params: {}, primaryOnly: true };
227
230
  case "cluster-init": {
228
231
  if (!IP_LIST.test(rest[1])) throw new UsageError("cluster-init needs a comma separated list of IP addresses");
229
232
  return {
@@ -35,5 +35,6 @@ case "$name" in
35
35
  printf '%s' "$lines" | grep -Eq '^[1-9][0-9]{0,3}$' || { echo "bad lines" >&2; exit 2; }
36
36
  tail -n "$lines" "$HOME/llama-$svc.log" ;;
37
37
  nexus-disk) df -h / ;;
38
+ macos-updates) [ -z "$rest" ] || { echo "macos-updates takes no arguments" >&2; exit 2; }; sw_vers -productVersion; softwareupdate -l 2>&1 | grep -E 'Title:' || true ;;
38
39
  *) echo "refused: $cmd" >&2; exit 2 ;;
39
40
  esac
@@ -39,6 +39,7 @@ export const INFERENCE_COMMANDS: Readonly<Record<string, RemoteSpec>> = {
39
39
  "nexus-stop": { args: 1, help: "nexus-stop <embed|rerank> --confirm launchctl bootout", timeoutMs: 2 * MINUTE, confirm: true },
40
40
  "nexus-log": { args: 2, help: "nexus-log <embed|rerank> <lines>", timeoutMs: MINUTE },
41
41
  "nexus-disk": { args: 0, help: "free space on the Nexus disk", timeoutMs: MINUTE },
42
+ "macos-updates": { args: 0, help: "installed macOS version and the updates Software Update offers", timeoutMs: 3 * MINUTE },
42
43
  };
43
44
 
44
45
  const NEXUS_SERVICES = ["embed", "rerank"];
@@ -216,6 +216,7 @@ describe("network wrapper", () => {
216
216
  describe("dns wrapper", () => {
217
217
  it("maps technitium commands to documented API paths", () => {
218
218
  expect(technitiumRequest("catalog.invalid", ["zones"])).toEqual({ path: "/api/zones/list", params: {}, primaryOnly: false });
219
+ expect(technitiumRequest("catalog.invalid", ["check-update"])).toEqual({ path: "/api/user/checkForUpdate", params: {}, primaryOnly: true });
219
220
  expect(technitiumRequest("catalog.invalid", ["zones", "*.37pla.net"])).toEqual({ path: "/api/zones/list", params: { filterName: "*.37pla.net" }, primaryOnly: false });
220
221
  expect(technitiumRequest("catalog.invalid", ["records", "37pla.net"])).toEqual({
221
222
  path: "/api/zones/records/get",