@astrofoundry/pi-astro 0.37.3 → 0.37.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/arcane/SKILL.md +3 -3
- package/specialists/arcane/run.ts +57 -18
- package/specialists/lib/arcane-host.test.ts +4 -1
- package/specialists/lib/arcane-manager.ts +8 -0
- package/specialists/lib/stack-update.test.ts +11 -2
- package/specialists/lib/stack-update.ts +9 -0
package/package.json
CHANGED
package/skills/arcane/SKILL.md
CHANGED
|
@@ -44,9 +44,9 @@ Messages that start with "Arcane Daily Report:" are the bridge's scheduled repor
|
|
|
44
44
|
|
|
45
45
|
Apply updates only when asked, one stack at a time, with `["stack-update", "<stack>"]`. It applies every pending update of the stack in one call, without judgment:
|
|
46
46
|
|
|
47
|
-
- Pinned tags (`edit tag`) and digests (`edit digest`) move in the stack's Compose file through the GitOps checkout: commit, push, then a wait of up to 20 minutes for
|
|
48
|
-
- `projects redeploy <stack> --yes` then pulls and restarts the stack (up to 35 minutes for large pulls).
|
|
49
|
-
- Checks: every service has a container that runs (a one-shot job may have exited 0), none is unhealthy or still starting after 5 minutes, and a fresh check finds every updated image current.
|
|
47
|
+
- Pinned tags (`edit tag`) and digests (`edit digest`) move in the stack's Compose file through the GitOps checkout: commit, push, start the stack's GitOps sync at once (`gitops sync`, so a long sync interval does not matter), then a wait of up to 20 minutes for it to report success at that commit. Only `image:` lines that name exactly the old reference change.
|
|
48
|
+
- `projects redeploy <stack> --yes` then pulls and restarts the stack (up to 35 minutes for large pulls). Arcane can leave a running container on its old image although the pull moved its tag (Docker then lists it by image id); each such container is recreated with `containers redeploy <id>` and named in `recreated`.
|
|
49
|
+
- Checks: every service has a container that runs (a one-shot job may have exited 0), found through its Compose project, none still runs an older image than its tag, none is unhealthy or still starting after 5 minutes, and a fresh check finds every updated image current.
|
|
50
50
|
- A failed check after a Compose change reverts the commit, pushes, waits for the sync, and redeploys the previous file. A pull alone cannot be undone; say so when it fails.
|
|
51
51
|
|
|
52
52
|
The answer is JSON with `status`: `updated`, `rolled back`, `failed` (with `reason`), or `nothing to do`, plus `services` (`service`, `from`, `to`, `applied`: `pull` or the new reference, `result`: `current`, `behind`, `unchecked`) and `commit`/`revert` when the Compose file changed. Report it in plain words.
|
|
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
|
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
5
5
|
import { buildHostRemote, compareAudit, HOST_COMMANDS, type AuditFile } from "../lib/arcane-host.ts";
|
|
6
6
|
import { type CommitVersion, coveringOperation, judgeChange, parseChangeQuery, readTail } from "../lib/change-origin.ts";
|
|
7
|
-
import { failedChecks, MANAGER_COMPOSE, MANAGER_IMAGE, normalizeVersion, olderThan, pinCompose, projectSync, syncReady, utcStamp, type VerifyCheck } from "../lib/arcane-manager.ts";
|
|
7
|
+
import { failedChecks, MANAGER_COMPOSE, MANAGER_IMAGE, normalizeVersion, olderThan, pinCompose, projectSync, syncId, syncReady, utcStamp, type VerifyCheck } from "../lib/arcane-manager.ts";
|
|
8
8
|
import { readConfig } from "../lib/config.ts";
|
|
9
9
|
import { isRecent, listContainer, type WatchedContainer, withDetail } from "../lib/container-watch.ts";
|
|
10
10
|
import { containerFacts, deploymentChecks, listRows, parseDeploymentArgs, syncFacts, type DeploymentExpectation } from "../lib/deployment.ts";
|
|
@@ -18,7 +18,7 @@ import { run, type RunResult } from "../lib/proc.ts";
|
|
|
18
18
|
import { HttpRegistry, parseImageRef } from "../lib/registry.ts";
|
|
19
19
|
import { releaseNotesFor } from "../lib/releases.ts";
|
|
20
20
|
import { ensureCheckout, git, REPO_HELP, repoCommand, repoStatus, safeRepoPath, type RepoSpec } from "../lib/repo.ts";
|
|
21
|
-
import { commitMessage, containersProblem, crossesMajor, HOLDS_FILE, nextImageRef, parseHolds, pendingRows, replaceImageRef, type ServiceResult, type StackContainer, type StackStatus } from "../lib/stack-update.ts";
|
|
21
|
+
import { commitMessage, containersProblem, crossesMajor, HOLDS_FILE, isStaleImage, nextImageRef, parseHolds, pendingRows, replaceImageRef, type ServiceResult, type StackContainer, type StackStatus } from "../lib/stack-update.ts";
|
|
22
22
|
import { ARCANE_DENIED_KEYS, filterJsonText, stripKeys } from "../lib/sanitize.ts";
|
|
23
23
|
import { readSecret, secretPath } from "../lib/secrets.ts";
|
|
24
24
|
import { sshFixed } from "../lib/ssh.ts";
|
|
@@ -107,8 +107,9 @@ ${Object.entries(HOST_COMMANDS)
|
|
|
107
107
|
the held stacks of ${HOLDS_FILE}. Checks only: never pulls images or redeploys.
|
|
108
108
|
stack-update <stack> [--confirm]
|
|
109
109
|
apply every pending update of one stack: pinned tags and digests move in the stack's
|
|
110
|
-
Compose file (commit, push, wait for
|
|
111
|
-
|
|
110
|
+
Compose file (commit, push, start the sync and wait for it), then the stack is redeployed
|
|
111
|
+
with a pull, containers the redeploy left on an old image are recreated one by one, and
|
|
112
|
+
the stack is checked (containers running and healthy, images current). A failed check after a
|
|
112
113
|
Compose change reverts it and redeploys; a pull alone cannot be undone. Refuses held
|
|
113
114
|
stacks, stopped stacks, and arcane (manager-upgrade). An update across a major version
|
|
114
115
|
needs --confirm, which asks an owner for approval.
|
|
@@ -588,6 +589,8 @@ async function managerRollback(args: string[]): Promise<number> {
|
|
|
588
589
|
const STACK_SYNC_WAIT_MS = 20 * MINUTE;
|
|
589
590
|
/** arcane-cli allows a project redeploy up to 30 minutes for large pulls. */
|
|
590
591
|
const REDEPLOY_TIMEOUT_MS = 35 * MINUTE;
|
|
592
|
+
/** `gitops sync` answers once the files are copied, about a minute even with a pre-deploy hook. */
|
|
593
|
+
const SYNC_START_TIMEOUT_MS = 10 * MINUTE;
|
|
591
594
|
/** Containers with a health check get this long to leave "starting". */
|
|
592
595
|
const HEALTH_WAIT_MS = 5 * MINUTE;
|
|
593
596
|
|
|
@@ -620,22 +623,58 @@ async function stackState(cli: Cli, stack: string, startedAt: number): Promise<S
|
|
|
620
623
|
return { rows, services: services.length, stopped: services.some((s) => s.stopped), composePath };
|
|
621
624
|
}
|
|
622
625
|
|
|
623
|
-
/**
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
626
|
+
/**
|
|
627
|
+
* The stack's containers as `containers get` reports them, kept when Compose
|
|
628
|
+
* says they belong to it. Candidates are the containers listed with one of the
|
|
629
|
+
* stack's image references, or with a bare image id: Docker shows that once a
|
|
630
|
+
* pull moved the container's tag to a newer image it does not run.
|
|
631
|
+
*/
|
|
632
|
+
async function stackContainers(cli: Cli, stack: string, refs: readonly string[]): Promise<(StackContainer & { id: string })[]> {
|
|
633
|
+
const listed = listRows(await cliJson(cli, ["containers", "list", "--all"])).filter((c) => (typeof c.image === "string" && refs.includes(c.image)) || isStaleImage(c.image));
|
|
634
|
+
const out: (StackContainer & { id: string })[] = [];
|
|
627
635
|
for (const c of listed) {
|
|
628
636
|
if (typeof c.id !== "string") continue;
|
|
629
637
|
const name = Array.isArray(c.names) && typeof c.names[0] === "string" ? c.names[0].replace(/^\//, "") : c.id.slice(0, 12);
|
|
630
638
|
const facts = containerFacts(await cliJson(cli, ["containers", "get", c.id]), name);
|
|
631
|
-
if (facts.project === stack) out.push({ name, running: facts.running, state: facts.state, exitCode: facts.exitCode, health: facts.health });
|
|
639
|
+
if (facts.project === stack) out.push({ id: c.id, name, running: facts.running, state: facts.state, exitCode: facts.exitCode, health: facts.health, stale: isStaleImage(c.image) });
|
|
632
640
|
}
|
|
633
641
|
return out;
|
|
634
642
|
}
|
|
635
643
|
|
|
636
|
-
|
|
644
|
+
async function stackRefs(cli: Cli, stack: string): Promise<string[]> {
|
|
645
|
+
return stackServices(await cliJson(cli, ["projects", "list", "--all"]))
|
|
646
|
+
.filter((s) => s.stack === stack)
|
|
647
|
+
.map((s) => s.ref)
|
|
648
|
+
.filter((r): r is string => r !== null);
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
/**
|
|
652
|
+
* Arcane's project redeploy can leave a running container on its old image
|
|
653
|
+
* although the pull moved its tag (paperless-db and paperless-redis on
|
|
654
|
+
* 2026-10-22). Arcane's container redeploy pulls and recreates that one
|
|
655
|
+
* container through Compose; returns the names it recreated, or the error.
|
|
656
|
+
*/
|
|
657
|
+
async function recreateStale(config: ArcaneConfig, stack: string): Promise<{ recreated: string[]; problem: string | null }> {
|
|
658
|
+
return withCli(config, async (cli) => {
|
|
659
|
+
const stale = (await stackContainers(cli, stack, await stackRefs(cli, stack))).filter((c) => c.running === true && c.stale);
|
|
660
|
+
const recreated: string[] = [];
|
|
661
|
+
for (const c of stale) {
|
|
662
|
+
const result = await cli(["containers", "redeploy", c.id], REDEPLOY_TIMEOUT_MS);
|
|
663
|
+
if (result.code !== 0) return { recreated, problem: `redeploy of ${c.name} failed: ${(result.stderr.trim().split("\n").pop() ?? "").slice(0, 200) || `exited ${result.code}`}` };
|
|
664
|
+
recreated.push(c.name);
|
|
665
|
+
}
|
|
666
|
+
return { recreated, problem: null };
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* Starts the stack's sync at once, so its interval (hourly for profiles) does
|
|
672
|
+
* not matter, then waits until it reports success at `commit`; false when it
|
|
673
|
+
* does not within the limit. A failed start leaves the scheduled sync to it.
|
|
674
|
+
*/
|
|
637
675
|
async function waitForSync(config: ArcaneConfig, stack: string, commit: string): Promise<{ ok: boolean; last: string }> {
|
|
638
676
|
const deadline = Date.now() + STACK_SYNC_WAIT_MS;
|
|
677
|
+
await withCli(config, async (cli) => cli(["gitops", "sync", syncId(await cliJson(cli, ["projects", "list", "--all"]), await cliJson(cli, ["gitops", "list", "--all"]), stack)], SYNC_START_TIMEOUT_MS)).catch(() => undefined);
|
|
639
678
|
for (;;) {
|
|
640
679
|
const sync = await withCli(config, async (cli) => projectSync(await cliJson(cli, ["projects", "list", "--all"]), await cliJson(cli, ["gitops", "list", "--all"]), stack));
|
|
641
680
|
const sameCommit = sync.commit !== null && (commit.startsWith(sync.commit) || sync.commit.startsWith(commit));
|
|
@@ -654,13 +693,7 @@ async function redeploy(config: ArcaneConfig, stack: string): Promise<string | n
|
|
|
654
693
|
async function settle(config: ArcaneConfig, stack: string, services: number): Promise<string | null> {
|
|
655
694
|
const deadline = Date.now() + HEALTH_WAIT_MS;
|
|
656
695
|
for (;;) {
|
|
657
|
-
const problem = await withCli(config, async (cli) =>
|
|
658
|
-
const refs = stackServices(await cliJson(cli, ["projects", "list", "--all"]))
|
|
659
|
-
.filter((s) => s.stack === stack)
|
|
660
|
-
.map((s) => s.ref)
|
|
661
|
-
.filter((r): r is string => r !== null);
|
|
662
|
-
return containersProblem(await stackContainers(cli, stack, refs), services);
|
|
663
|
-
});
|
|
696
|
+
const problem = await withCli(config, async (cli) => containersProblem(await stackContainers(cli, stack, await stackRefs(cli, stack)), services));
|
|
664
697
|
if (problem === null || Date.now() > deadline) return problem;
|
|
665
698
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
666
699
|
}
|
|
@@ -726,6 +759,12 @@ async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number
|
|
|
726
759
|
}
|
|
727
760
|
|
|
728
761
|
let problem = await redeploy(config, stack);
|
|
762
|
+
let recreated: string[] = [];
|
|
763
|
+
if (problem === null) {
|
|
764
|
+
const stale = await recreateStale(config, stack);
|
|
765
|
+
recreated = stale.recreated;
|
|
766
|
+
problem = stale.problem;
|
|
767
|
+
}
|
|
729
768
|
if (problem === null) problem = await settle(config, stack, before.services);
|
|
730
769
|
if (problem === null) {
|
|
731
770
|
const after = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
|
|
@@ -738,7 +777,7 @@ async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number
|
|
|
738
777
|
if (behind.length > 0) problem = `still behind after the redeploy: ${behind.join(", ")}`;
|
|
739
778
|
}
|
|
740
779
|
if (problem === null) {
|
|
741
|
-
answer("updated", { services: results, ...(commit ? { commit } : {}) });
|
|
780
|
+
answer("updated", { services: results, ...(commit ? { commit } : {}), ...(recreated.length > 0 ? { recreated } : {}) });
|
|
742
781
|
return 0;
|
|
743
782
|
}
|
|
744
783
|
if (commit === null) {
|
|
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
|
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { afterEach, describe, expect, it } from "vitest";
|
|
6
6
|
import { buildHostRemote, compareAudit, HOST_COMMANDS } from "./arcane-host.ts";
|
|
7
|
-
import { failedChecks, normalizeVersion, olderThan, pinCompose, pinnedDigest, projectSync, syncReady, utcStamp } from "./arcane-manager.ts";
|
|
7
|
+
import { failedChecks, normalizeVersion, olderThan, pinCompose, pinnedDigest, projectSync, syncId, syncReady, utcStamp } from "./arcane-manager.ts";
|
|
8
8
|
import { containerFacts, deploymentChecks, listRows, parseDeploymentArgs, syncFacts } from "./deployment.ts";
|
|
9
9
|
import { UsageError } from "./errors.ts";
|
|
10
10
|
|
|
@@ -130,6 +130,9 @@ describe("manager upgrade helpers", () => {
|
|
|
130
130
|
expect(syncReady({ ...sync, status: "success" }, "abc1234def", "x", "y")).toBe(false);
|
|
131
131
|
expect(syncReady({ ...sync, status: "success", commit: "fff0000" }, "abc1234def", "x", "x")).toBe(false);
|
|
132
132
|
expect(() => projectSync(projects, { data: [] }, "arcane")).toThrow(UsageError);
|
|
133
|
+
expect(syncId(projects, { data: [{ id: "s1", projectId: "p1" }] }, "arcane")).toBe("s1");
|
|
134
|
+
expect(() => syncId(projects, { data: [{ id: "s1", projectId: "p1" }, { id: "s2", projectId: "p1" }] }, "arcane")).toThrow(UsageError);
|
|
135
|
+
expect(() => syncId(projects, syncs, "other")).toThrow(UsageError);
|
|
133
136
|
expect(failedChecks([{ check: "version", ok: false, observed: "2.16.1" }, { check: "digest", ok: true }])).toBe('version ("2.16.1")');
|
|
134
137
|
});
|
|
135
138
|
});
|
|
@@ -56,6 +56,14 @@ export function projectSync(projects: unknown, syncs: unknown, name: string): Sy
|
|
|
56
56
|
return { status: text(s.lastSyncStatus), commit: text(s.lastSyncCommit), at: text(s.lastSyncAt) };
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
/** The id of the project's one GitOps sync, for `gitops sync <id>`. */
|
|
60
|
+
export function syncId(projects: unknown, syncs: unknown, name: string): string {
|
|
61
|
+
const project = listRows(projects).filter((p) => p.name === name);
|
|
62
|
+
const mine = project.length === 1 ? listRows(syncs).filter((s) => s.projectId === project[0].id) : [];
|
|
63
|
+
if (mine.length !== 1 || typeof mine[0].id !== "string") throw new UsageError(`project ${name} needs exactly one GitOps sync`);
|
|
64
|
+
return mine[0].id;
|
|
65
|
+
}
|
|
66
|
+
|
|
59
67
|
/**
|
|
60
68
|
* Ready for the host step when the sync reports success at the pin commit and
|
|
61
69
|
* the deployed Compose file is the committed one. A sync that copied the file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import type { UpdateRow } from "./image-updates.ts";
|
|
3
|
-
import { commitMessage, containersProblem, crossesMajor, nextImageRef, parseHolds, pendingRows, replaceImageRef, type StackContainer } from "./stack-update.ts";
|
|
3
|
+
import { commitMessage, containersProblem, crossesMajor, isStaleImage, nextImageRef, parseHolds, pendingRows, replaceImageRef, type StackContainer } from "./stack-update.ts";
|
|
4
4
|
|
|
5
5
|
const A = `sha256:${"a".repeat(64)}`;
|
|
6
6
|
const B = `sha256:${"b".repeat(64)}`;
|
|
@@ -80,7 +80,7 @@ describe("Compose rewrite", () => {
|
|
|
80
80
|
});
|
|
81
81
|
|
|
82
82
|
describe("stack health after a redeploy", () => {
|
|
83
|
-
const up = (name: string, over: Partial<StackContainer> = {}): StackContainer => ({ name, running: true, state: "running", exitCode: null, health: "healthy", ...over });
|
|
83
|
+
const up = (name: string, over: Partial<StackContainer> = {}): StackContainer => ({ name, running: true, state: "running", exitCode: null, health: "healthy", stale: false, ...over });
|
|
84
84
|
|
|
85
85
|
it("passes running containers and finished one-shot jobs", () => {
|
|
86
86
|
expect(containersProblem([up("db"), up("app", { health: "none" }), up("init", { running: false, state: "exited", exitCode: 0 })], 3)).toBeNull();
|
|
@@ -91,6 +91,15 @@ describe("stack health after a redeploy", () => {
|
|
|
91
91
|
expect(containersProblem([up("db"), up("init", { running: false, state: "exited", exitCode: 1 })], 2)).toBe("not running: init");
|
|
92
92
|
expect(containersProblem([up("db", { health: "unhealthy" }), up("app", { health: "starting" })], 2)).toBe("health: db (unhealthy), app (starting)");
|
|
93
93
|
});
|
|
94
|
+
|
|
95
|
+
it("names running containers left on an older image than their tag, not finished jobs", () => {
|
|
96
|
+
expect(containersProblem([up("db", { stale: true }), up("redis", { stale: true }), up("app")], 3)).toBe("still on an older image than its tag: db, redis");
|
|
97
|
+
expect(containersProblem([up("init", { running: false, state: "exited", exitCode: 0, stale: true })], 1)).toBeNull();
|
|
98
|
+
expect(isStaleImage(A)).toBe(true);
|
|
99
|
+
expect(isStaleImage("postgres:18")).toBe(false);
|
|
100
|
+
expect(isStaleImage(`postgres@${A}`)).toBe(false);
|
|
101
|
+
expect(isStaleImage(undefined)).toBe(false);
|
|
102
|
+
});
|
|
94
103
|
});
|
|
95
104
|
|
|
96
105
|
describe("commit message", () => {
|
|
@@ -95,6 +95,13 @@ export interface StackContainer {
|
|
|
95
95
|
state: string | null;
|
|
96
96
|
exitCode: number | null;
|
|
97
97
|
health: string;
|
|
98
|
+
/** Docker lists the container by image id: its Compose tag now names a newer image that it does not run. */
|
|
99
|
+
stale: boolean;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** A container's listed image is a bare id once its tag moved to another image. */
|
|
103
|
+
export function isStaleImage(image: unknown): boolean {
|
|
104
|
+
return typeof image === "string" && /^sha256:[0-9a-f]{64}$/.test(image);
|
|
98
105
|
}
|
|
99
106
|
|
|
100
107
|
/**
|
|
@@ -105,6 +112,8 @@ export function containersProblem(containers: readonly StackContainer[], expecte
|
|
|
105
112
|
if (containers.length < expected) return `${containers.length} of ${expected} services have a container`;
|
|
106
113
|
const down = containers.filter((c) => c.running !== true && !(c.state === "exited" && c.exitCode === 0)).map((c) => c.name);
|
|
107
114
|
if (down.length > 0) return `not running: ${down.join(", ")}`;
|
|
115
|
+
const old = containers.filter((c) => c.running === true && c.stale).map((c) => c.name);
|
|
116
|
+
if (old.length > 0) return `still on an older image than its tag: ${old.join(", ")}`;
|
|
108
117
|
const sick = containers.filter((c) => c.health === "unhealthy" || c.health === "starting").map((c) => `${c.name} (${c.health})`);
|
|
109
118
|
return sick.length > 0 ? `health: ${sick.join(", ")}` : null;
|
|
110
119
|
}
|