@rizom/ops 0.2.0-alpha.26 → 0.2.0-alpha.261
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -2
- package/dist/brains-ops.js +407 -313
- package/dist/cert-bootstrap.d.ts +3 -1
- package/dist/content-repo-ref.d.ts +10 -0
- package/dist/content-repo.d.ts +1 -0
- package/dist/deploy.js +99 -166
- package/dist/directory-sync-stress-system.d.ts +63 -0
- package/dist/directory-sync-stress.d.ts +105 -0
- package/dist/entries/deploy.d.ts +3 -2
- package/dist/images.d.ts +76 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +443 -308
- package/dist/legacy-pilot-migration.d.ts +11 -0
- package/dist/load-registry.d.ts +55 -6
- package/dist/observed-status.d.ts +1 -1
- package/dist/origin-ca.d.ts +1 -1
- package/dist/parse-args.d.ts +2 -10
- package/dist/preview-domain.d.ts +9 -0
- package/dist/push-secrets.d.ts +2 -9
- package/dist/push-target.d.ts +1 -2
- package/dist/reconcile-dry-run.d.ts +9 -0
- package/dist/run-command.d.ts +8 -3
- package/dist/run-subprocess.d.ts +1 -6
- package/dist/schema.d.ts +100 -162
- package/dist/secrets-encrypt.d.ts +7 -13
- package/dist/ssh-key-bootstrap.d.ts +1 -26
- package/dist/stage-legacy-crossover.d.ts +20 -0
- package/dist/user-add.d.ts +15 -0
- package/dist/verify-user.d.ts +19 -0
- package/package.json +46 -42
- package/templates/rover-pilot/.env.schema +24 -3
- package/templates/rover-pilot/.github/actions/varlock-env/action.yml +47 -0
- package/templates/rover-pilot/.github/workflows/build.yml +64 -17
- package/templates/rover-pilot/.github/workflows/deploy.yml +70 -60
- package/templates/rover-pilot/.github/workflows/directory-sync-stress.yml +100 -0
- package/templates/rover-pilot/.github/workflows/reconcile.yml +10 -4
- package/templates/rover-pilot/README.md +13 -6
- package/templates/rover-pilot/deploy/scripts/decrypt-user-secrets.ts +80 -24
- package/templates/rover-pilot/deploy/scripts/helpers.ts +3 -0
- package/templates/rover-pilot/deploy/scripts/install-health-watchdog.ts +144 -0
- package/templates/rover-pilot/deploy/scripts/resolve-deploy-handles.ts +12 -3
- package/templates/rover-pilot/deploy/scripts/resolve-missing-images.ts +13 -0
- package/templates/rover-pilot/deploy/scripts/resolve-user-config.ts +44 -9
- package/templates/rover-pilot/deploy/scripts/sync-content-repo.ts +51 -47
- package/templates/rover-pilot/deploy/scripts/update-dns.ts +14 -4
- package/templates/rover-pilot/deploy/scripts/validate-secrets.ts +12 -1
- package/templates/rover-pilot/docs/canonical-crossover-record.md +104 -0
- package/templates/rover-pilot/docs/onboarding-checklist.md +29 -13
- package/templates/rover-pilot/docs/operator-playbook.md +234 -24
- package/templates/rover-pilot/docs/user-onboarding.md +49 -343
- package/templates/rover-pilot/pilot.yaml +3 -4
- package/templates/rover-pilot/.kamal/hooks/pre-deploy +0 -9
- package/templates/rover-pilot/deploy/Dockerfile +0 -30
- package/templates/rover-pilot/deploy/kamal/deploy.yml +0 -40
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
name: Directory Sync Stress
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
inputs:
|
|
6
|
+
handle:
|
|
7
|
+
description: Smoke handle only
|
|
8
|
+
required: true
|
|
9
|
+
type: string
|
|
10
|
+
default: smoke
|
|
11
|
+
profile:
|
|
12
|
+
description: Workload profile
|
|
13
|
+
required: true
|
|
14
|
+
type: choice
|
|
15
|
+
default: regression
|
|
16
|
+
options:
|
|
17
|
+
- regression
|
|
18
|
+
- load
|
|
19
|
+
- stress
|
|
20
|
+
confirm:
|
|
21
|
+
description: Type stress:<handle> to confirm reversible content writes
|
|
22
|
+
required: true
|
|
23
|
+
type: string
|
|
24
|
+
|
|
25
|
+
permissions:
|
|
26
|
+
contents: read
|
|
27
|
+
|
|
28
|
+
concurrency:
|
|
29
|
+
group: directory-sync-stress-${{ inputs.handle }}
|
|
30
|
+
cancel-in-progress: false
|
|
31
|
+
|
|
32
|
+
jobs:
|
|
33
|
+
stress:
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
timeout-minutes: 75
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/checkout@v5
|
|
38
|
+
|
|
39
|
+
- uses: oven-sh/setup-bun@v2
|
|
40
|
+
|
|
41
|
+
- name: Install operator tooling
|
|
42
|
+
run: bun install
|
|
43
|
+
|
|
44
|
+
- name: Load Bitwarden environment through Varlock
|
|
45
|
+
uses: ./.github/actions/varlock-env
|
|
46
|
+
env:
|
|
47
|
+
BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
|
|
48
|
+
|
|
49
|
+
- name: Run smoke directory-sync stress profile
|
|
50
|
+
env:
|
|
51
|
+
HANDLE_INPUT: ${{ inputs.handle }}
|
|
52
|
+
CONFIRM_INPUT: ${{ inputs.confirm }}
|
|
53
|
+
run: |
|
|
54
|
+
bunx brains-ops stress:directory-sync "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
|
|
55
|
+
--profile "${{ inputs.profile }}" \
|
|
56
|
+
--confirm "$CONFIRM_INPUT" \
|
|
57
|
+
--artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
|
|
58
|
+
|
|
59
|
+
- name: Upload stress evidence
|
|
60
|
+
if: always()
|
|
61
|
+
uses: actions/upload-artifact@v4
|
|
62
|
+
with:
|
|
63
|
+
name: directory-sync-stress-${{ inputs.handle }}-${{ inputs.profile }}-${{ github.run_id }}
|
|
64
|
+
path: ${{ runner.temp }}/directory-sync-stress
|
|
65
|
+
if-no-files-found: warn
|
|
66
|
+
|
|
67
|
+
cleanup:
|
|
68
|
+
needs: stress
|
|
69
|
+
if: always()
|
|
70
|
+
runs-on: ubuntu-latest
|
|
71
|
+
timeout-minutes: 20
|
|
72
|
+
steps:
|
|
73
|
+
- uses: actions/checkout@v5
|
|
74
|
+
|
|
75
|
+
- uses: oven-sh/setup-bun@v2
|
|
76
|
+
|
|
77
|
+
- name: Install operator tooling
|
|
78
|
+
run: bun install
|
|
79
|
+
|
|
80
|
+
- name: Load Bitwarden environment through Varlock
|
|
81
|
+
uses: ./.github/actions/varlock-env
|
|
82
|
+
env:
|
|
83
|
+
BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
|
|
84
|
+
|
|
85
|
+
- name: Remove residual stress probes
|
|
86
|
+
env:
|
|
87
|
+
HANDLE_INPUT: ${{ inputs.handle }}
|
|
88
|
+
CONFIRM_INPUT: ${{ inputs.confirm }}
|
|
89
|
+
run: |
|
|
90
|
+
bunx brains-ops stress:directory-sync:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
|
|
91
|
+
--confirm "$CONFIRM_INPUT" \
|
|
92
|
+
--artifacts-dir "$RUNNER_TEMP/directory-sync-stress-cleanup"
|
|
93
|
+
|
|
94
|
+
- name: Upload cleanup evidence
|
|
95
|
+
if: always()
|
|
96
|
+
uses: actions/upload-artifact@v4
|
|
97
|
+
with:
|
|
98
|
+
name: directory-sync-stress-cleanup-${{ inputs.handle }}-${{ github.run_id }}
|
|
99
|
+
path: ${{ runner.temp }}/directory-sync-stress-cleanup
|
|
100
|
+
if-no-files-found: warn
|
|
@@ -35,21 +35,27 @@ jobs:
|
|
|
35
35
|
|
|
36
36
|
- name: Commit generated outputs
|
|
37
37
|
run: |
|
|
38
|
-
|
|
38
|
+
# A newly added user's generated directory is untracked, and
|
|
39
|
+
# `git diff` cannot see untracked files — without intent-to-add
|
|
40
|
+
# the new config is silently dropped here and every later deploy
|
|
41
|
+
# skips that user (their files never appear in a commit range).
|
|
42
|
+
git add --intent-to-add -- users
|
|
43
|
+
|
|
44
|
+
if git diff --quiet -- users; then
|
|
39
45
|
exit 0
|
|
40
46
|
fi
|
|
41
47
|
|
|
42
48
|
git fetch origin "${{ github.ref_name }}"
|
|
43
|
-
if git diff --quiet "origin/${{ github.ref_name }}" --
|
|
49
|
+
if git diff --quiet "origin/${{ github.ref_name }}" -- users; then
|
|
44
50
|
exit 0
|
|
45
51
|
fi
|
|
46
52
|
|
|
47
53
|
patch_file="$(mktemp)"
|
|
48
|
-
git diff --binary --
|
|
54
|
+
git diff --binary -- users > "$patch_file"
|
|
49
55
|
git reset --hard "origin/${{ github.ref_name }}"
|
|
50
56
|
git apply --3way --index "$patch_file"
|
|
51
57
|
|
|
52
|
-
if git diff --cached --quiet --
|
|
58
|
+
if git diff --cached --quiet -- users; then
|
|
53
59
|
exit 0
|
|
54
60
|
fi
|
|
55
61
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
# private brain pilot
|
|
2
2
|
|
|
3
|
-
Private desired-state
|
|
3
|
+
Private desired-state repository for the hosted brain pilot. The generated repository/service name may remain `rover-pilot` for deployment identity compatibility.
|
|
4
4
|
|
|
5
5
|
This is a single operator-owned repo. Pilot users do not get their own brain repos.
|
|
6
6
|
Per-user deploy config lives under `users/<handle>/`, while content stays in per-user content repos.
|
|
@@ -29,18 +29,25 @@ The repo also checks in its deploy contract:
|
|
|
29
29
|
- `.github/workflows/*`
|
|
30
30
|
|
|
31
31
|
`.env.schema` is the single source of truth for required and sensitive deploy vars.
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
Use separate GitHub tokens: `CONTENT_REPO_ADMIN_TOKEN` for operator-side content repo creation/checks, and `GIT_SYNC_TOKEN` for runtime directory-sync git access.
|
|
33
|
+
The default pilot image tag is `brain-${brainVersion}` end to end. A user with `siteOverride` gets an isolated `brain-${brainVersion}-sites-${packageHash}` image instead.
|
|
34
|
+
When the effective brain version (`pilot.yaml.brainVersion`, or a cohort override) changes and you push, CI rebuilds the required default/site tags, refreshes generated user env files, and redeploys affected users. Every external site and theme package uses its own required exact version pin; ops never infers either version from the brain or from another package.
|
|
34
35
|
When a push changes only deploy contract files, CI prints `No affected user configs; skipping deploy.` and stops before Kamal.
|
|
35
36
|
|
|
36
37
|
## Commands
|
|
37
38
|
|
|
38
39
|
- `brains-ops init <repo>`
|
|
39
|
-
- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health`, and unauthenticated `/mcp` status checks
|
|
40
|
-
- `brains-ops
|
|
40
|
+
- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health/ready`, and unauthenticated `/mcp` status checks
|
|
41
|
+
- `brains-ops user:add <repo> <handle> --cohort <cohort>` — scaffolds a user file, per-user secrets template, and cohort membership
|
|
42
|
+
- `brains-ops onboard <repo> <handle>` — creates/seeds the user's content repo with separate admin and sync tokens
|
|
41
43
|
- `brains-ops age-key:bootstrap <repo>`
|
|
42
44
|
- `brains-ops ssh-key:bootstrap <repo>`
|
|
43
45
|
- `brains-ops cert:bootstrap <repo>`
|
|
44
46
|
- `brains-ops secrets:encrypt <repo> <handle>`
|
|
45
47
|
- `brains-ops reconcile-cohort <repo> <cohort>`
|
|
46
48
|
- `brains-ops reconcile-all <repo>`
|
|
49
|
+
- `brains-ops reconcile-all <repo> --dry-run` (isolated, no external content-repository access; lists both passes' changed files)
|
|
50
|
+
|
|
51
|
+
`render` owns the observational `views/users.md` projection. Reconcile owns generated per-user config and never rewrites observed status rows.
|
|
52
|
+
|
|
53
|
+
Use `docs/canonical-crossover-record.md` to record exact forward and rollback artifact pins before an approved crossover window.
|
|
@@ -15,13 +15,52 @@ const decrypter = new Decrypter();
|
|
|
15
15
|
decrypter.addIdentity(ageSecretKey);
|
|
16
16
|
|
|
17
17
|
const plaintext = await decrypter.decrypt(decoded, "text");
|
|
18
|
-
const secrets =
|
|
19
|
-
const pilot =
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
18
|
+
const secrets = parseStringYamlMapping(plaintext);
|
|
19
|
+
const pilot = parseStringYamlMapping(readFileSync("pilot.yaml", "utf8"));
|
|
20
|
+
|
|
21
|
+
writeSecretGitHubEnv("AI_API_KEY", secrets["aiApiKey"]);
|
|
22
|
+
writeSecretGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"]);
|
|
23
|
+
writeSecretGitHubEnv(
|
|
24
|
+
"CMS_CONTENT_REPO_PAT",
|
|
25
|
+
secrets["cmsContentRepoPat"] ?? secrets["gitSyncToken"],
|
|
26
|
+
);
|
|
27
|
+
writeSecretGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"]);
|
|
28
|
+
writeSecretGitHubEnv("DISCORD_PUBLIC_KEY", secrets["discordPublicKey"]);
|
|
29
|
+
writeSecretGitHubEnv("DISCORD_APPLICATION_ID", secrets["discordApplicationId"]);
|
|
30
|
+
// Per-user AT Protocol publishing credential (optional; from the user's
|
|
31
|
+
// encrypted secrets file). The scaffold wires this so a pilot can publish its
|
|
32
|
+
// brain's agent card to its PDS; a deployment that doesn't publish simply
|
|
33
|
+
// leaves it unset.
|
|
34
|
+
writeSecretGitHubEnv("ATPROTO_APP_PASSWORD", secrets["atprotoAppPassword"]);
|
|
35
|
+
// Per-user custom-domain TLS overrides the shared fleet certificate when a
|
|
36
|
+
// complete certificate/key pair is present in the encrypted user secrets.
|
|
37
|
+
const certificatePem = decodeEscapedSecret(secrets["certificatePem"]);
|
|
38
|
+
const privateKeyPem = decodeEscapedSecret(secrets["privateKeyPem"]);
|
|
39
|
+
if (Boolean(certificatePem) !== Boolean(privateKeyPem)) {
|
|
40
|
+
throw new Error(
|
|
41
|
+
"Custom-domain TLS secrets require both certificatePem and privateKeyPem",
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
// A corrupted stored value would otherwise only surface as a kamal-proxy
|
|
45
|
+
// "unable to load certificate" failure after the container has already booted.
|
|
46
|
+
assertPemShape("certificatePem", certificatePem);
|
|
47
|
+
assertPemShape("privateKeyPem", privateKeyPem);
|
|
48
|
+
writeSecretGitHubEnv("CERTIFICATE_PEM", certificatePem);
|
|
49
|
+
writeSecretGitHubEnv("PRIVATE_KEY_PEM", privateKeyPem);
|
|
50
|
+
|
|
51
|
+
function assertPemShape(name: string, value: string | undefined): void {
|
|
52
|
+
if (value === undefined || value.trim().length === 0) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
if (
|
|
56
|
+
!/-----BEGIN [A-Z0-9 ]+-----/.test(value) ||
|
|
57
|
+
!/-----END [A-Z0-9 ]+-----/.test(value)
|
|
58
|
+
) {
|
|
59
|
+
throw new Error(
|
|
60
|
+
`Secret ${name} is not PEM-shaped after unescaping; the stored value is corrupt`,
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
25
64
|
|
|
26
65
|
writeGitHubOutput(
|
|
27
66
|
"shared_ai_api_key_secret_name",
|
|
@@ -32,10 +71,29 @@ writeGitHubOutput(
|
|
|
32
71
|
requireFlatValue(pilot, "gitSyncToken", "pilot.yaml"),
|
|
33
72
|
);
|
|
34
73
|
writeGitHubOutput(
|
|
35
|
-
"
|
|
36
|
-
requireFlatValue(pilot, "
|
|
74
|
+
"shared_content_repo_admin_token_secret_name",
|
|
75
|
+
requireFlatValue(pilot, "contentRepoAdminToken", "pilot.yaml"),
|
|
37
76
|
);
|
|
38
77
|
|
|
78
|
+
function writeSecretGitHubEnv(name: string, value: string | undefined): void {
|
|
79
|
+
if (!value || value.trim().length === 0) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
maskGitHubSecret(value);
|
|
84
|
+
writeGitHubEnv(name, value);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function maskGitHubSecret(value: string): void {
|
|
88
|
+
const escaped = value
|
|
89
|
+
.replace(/%/g, "%25")
|
|
90
|
+
.replace(/\r/g, "%0D")
|
|
91
|
+
.replace(/\n/g, "%0A");
|
|
92
|
+
if (escaped.length > 0 && process.env["GITHUB_ACTIONS"] === "true") {
|
|
93
|
+
console.log(`::add-mask::${escaped}`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
39
97
|
function extractAgeIdentity(contents: string): string {
|
|
40
98
|
const line = contents
|
|
41
99
|
.split(/\r?\n/)
|
|
@@ -49,27 +107,25 @@ function extractAgeIdentity(contents: string): string {
|
|
|
49
107
|
return line;
|
|
50
108
|
}
|
|
51
109
|
|
|
52
|
-
function
|
|
53
|
-
const
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
if (!line || line.startsWith("#")) {
|
|
58
|
-
continue;
|
|
59
|
-
}
|
|
110
|
+
function parseStringYamlMapping(contents: string): Record<string, string> {
|
|
111
|
+
const parsed: unknown = Bun.YAML.parse(contents);
|
|
112
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
113
|
+
throw new Error("Expected a YAML mapping");
|
|
114
|
+
}
|
|
60
115
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
116
|
+
const result: Record<string, string> = {};
|
|
117
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
118
|
+
if (typeof value === "string") {
|
|
119
|
+
result[key] = value;
|
|
64
120
|
}
|
|
65
|
-
|
|
66
|
-
const [, key, rawValue] = match;
|
|
67
|
-
result[key] = rawValue.replace(/^['"]|['"]$/g, "");
|
|
68
121
|
}
|
|
69
|
-
|
|
70
122
|
return result;
|
|
71
123
|
}
|
|
72
124
|
|
|
125
|
+
function decodeEscapedSecret(value: string | undefined): string | undefined {
|
|
126
|
+
return value?.replace(/\\n/g, "\n");
|
|
127
|
+
}
|
|
128
|
+
|
|
73
129
|
function requireFlatValue(
|
|
74
130
|
values: Record<string, string>,
|
|
75
131
|
key: string,
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { requireEnv } from "./helpers";
|
|
2
|
+
|
|
3
|
+
const WATCHDOG_PATH = "/usr/local/sbin/brains-health-watchdog";
|
|
4
|
+
const SERVICE_PATH = "/etc/systemd/system/brains-health-watchdog.service";
|
|
5
|
+
const TIMER_PATH = "/etc/systemd/system/brains-health-watchdog.timer";
|
|
6
|
+
|
|
7
|
+
export const BRAIN_WATCHDOG_LABEL = "ai.rizom.brain.watchdog";
|
|
8
|
+
export const BRAIN_WATCHDOG_LABEL_VALUE = "true";
|
|
9
|
+
export const BRAIN_WATCHDOG_LABEL_FILTER: string = `${BRAIN_WATCHDOG_LABEL}=${BRAIN_WATCHDOG_LABEL_VALUE}`;
|
|
10
|
+
|
|
11
|
+
export const healthWatchdogScript: string = `#!/usr/bin/env bash
|
|
12
|
+
set -euo pipefail
|
|
13
|
+
umask 077
|
|
14
|
+
|
|
15
|
+
lock_path="\${BRAIN_WATCHDOG_LOCK_PATH:-/run/brains-health-watchdog.lock}"
|
|
16
|
+
exec 9>"$lock_path"
|
|
17
|
+
flock -n 9 || exit 0
|
|
18
|
+
|
|
19
|
+
incident_dir="\${BRAIN_WATCHDOG_INCIDENT_DIR:-/var/log/brains-health-watchdog}"
|
|
20
|
+
state_dir="\${BRAIN_WATCHDOG_STATE_DIR:-/var/lib/brains-health-watchdog}"
|
|
21
|
+
MAX_RESTARTS=3
|
|
22
|
+
WINDOW_SECONDS=3600
|
|
23
|
+
mkdir -p "$incident_dir" "$state_dir"
|
|
24
|
+
|
|
25
|
+
write_safe_inspect() {
|
|
26
|
+
docker inspect --format '{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Config.Image}},"state":{{json .State}},"restartPolicy":{{json .HostConfig.RestartPolicy}}}' "$1"
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
while read -r container_id; do
|
|
30
|
+
[ -n "$container_id" ] || continue
|
|
31
|
+
service_name=$(docker inspect --format '{{ index .Config.Labels "service" }}' "$container_id" 2>/dev/null || true)
|
|
32
|
+
container_name=$(docker inspect --format '{{ .Name }}' "$container_id" 2>/dev/null | tr -d '/' || true)
|
|
33
|
+
state_key=$(printf '%s-%s-%s' "$service_name" "$container_name" "$container_id" | tr -c 'A-Za-z0-9_.-' '_')
|
|
34
|
+
state_path="$state_dir/$state_key.restarts"
|
|
35
|
+
recent_path="$state_path.tmp"
|
|
36
|
+
now_epoch=$(date -u +%s)
|
|
37
|
+
cutoff_epoch=$((now_epoch - WINDOW_SECONDS))
|
|
38
|
+
touch "$state_path"
|
|
39
|
+
awk -v cutoff="$cutoff_epoch" '$1 >= cutoff' "$state_path" >"$recent_path"
|
|
40
|
+
restart_count=$(wc -l <"$recent_path")
|
|
41
|
+
|
|
42
|
+
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
|
|
43
|
+
incident_path="$incident_dir/$timestamp-$state_key.log"
|
|
44
|
+
temporary_path="$incident_path.tmp"
|
|
45
|
+
|
|
46
|
+
if [ "$restart_count" -ge "$MAX_RESTARTS" ]; then
|
|
47
|
+
{
|
|
48
|
+
printf '{"timestamp":"%s","event":"restart-suppressed","reason":"restart-budget-exhausted","service":"%s","container":"%s","restartsInWindow":%s}\\n' "$timestamp" "$service_name" "$container_name" "$restart_count"
|
|
49
|
+
write_safe_inspect "$container_id"
|
|
50
|
+
docker logs --timestamps --tail 1000 "$container_id"
|
|
51
|
+
} >"$temporary_path" 2>&1
|
|
52
|
+
mv "$temporary_path" "$incident_path"
|
|
53
|
+
mv "$recent_path" "$state_path"
|
|
54
|
+
logger -t brains-health-watchdog "restart budget exhausted container=$container_name service=$service_name incident=$incident_path"
|
|
55
|
+
continue
|
|
56
|
+
fi
|
|
57
|
+
|
|
58
|
+
{
|
|
59
|
+
printf '{"timestamp":"%s","event":"container-restart","reason":"docker-health-unhealthy","service":"%s","container":"%s"}\\n' "$timestamp" "$service_name" "$container_name"
|
|
60
|
+
write_safe_inspect "$container_id"
|
|
61
|
+
docker logs --timestamps --tail 1000 "$container_id"
|
|
62
|
+
} >"$temporary_path" 2>&1
|
|
63
|
+
mv "$temporary_path" "$incident_path"
|
|
64
|
+
printf '%s\\n' "$now_epoch" >>"$recent_path"
|
|
65
|
+
mv "$recent_path" "$state_path"
|
|
66
|
+
|
|
67
|
+
logger -t brains-health-watchdog "restarting unhealthy container=$container_name service=$service_name incident=$incident_path"
|
|
68
|
+
docker restart "$container_id"
|
|
69
|
+
done < <(docker ps --filter health=unhealthy --filter label=${BRAIN_WATCHDOG_LABEL_FILTER} --format '{{.ID}}')
|
|
70
|
+
`;
|
|
71
|
+
|
|
72
|
+
export const healthWatchdogServiceUnit: string = `[Unit]
|
|
73
|
+
Description=Restart persistently unhealthy Brain containers
|
|
74
|
+
After=docker.service
|
|
75
|
+
Requires=docker.service
|
|
76
|
+
|
|
77
|
+
[Service]
|
|
78
|
+
Type=oneshot
|
|
79
|
+
ExecStart=${WATCHDOG_PATH}
|
|
80
|
+
`;
|
|
81
|
+
|
|
82
|
+
export const healthWatchdogTimerUnit: string = `[Unit]
|
|
83
|
+
Description=Monitor Brain container liveness
|
|
84
|
+
|
|
85
|
+
[Timer]
|
|
86
|
+
OnBootSec=2min
|
|
87
|
+
OnUnitActiveSec=30s
|
|
88
|
+
RandomizedDelaySec=5s
|
|
89
|
+
Persistent=true
|
|
90
|
+
Unit=brains-health-watchdog.service
|
|
91
|
+
|
|
92
|
+
[Install]
|
|
93
|
+
WantedBy=timers.target
|
|
94
|
+
`;
|
|
95
|
+
|
|
96
|
+
function installCommand(path: string, content: string, mode: string): string {
|
|
97
|
+
const encoded = Buffer.from(content, "utf8").toString("base64");
|
|
98
|
+
return `printf '%s' '${encoded}' | $SUDO base64 --decode | $SUDO tee '${path}' >/dev/null\n$SUDO chmod ${mode} '${path}'`;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export function buildHealthWatchdogInstallScript(): string {
|
|
102
|
+
return `#!/usr/bin/env bash
|
|
103
|
+
set -euo pipefail
|
|
104
|
+
SUDO=""
|
|
105
|
+
if [ "$(id -u)" -ne 0 ]; then
|
|
106
|
+
SUDO=sudo
|
|
107
|
+
fi
|
|
108
|
+
|
|
109
|
+
${installCommand(WATCHDOG_PATH, healthWatchdogScript, "0755")}
|
|
110
|
+
${installCommand(SERVICE_PATH, healthWatchdogServiceUnit, "0644")}
|
|
111
|
+
${installCommand(TIMER_PATH, healthWatchdogTimerUnit, "0644")}
|
|
112
|
+
|
|
113
|
+
$SUDO systemctl daemon-reload
|
|
114
|
+
$SUDO systemctl enable --now brains-health-watchdog.timer
|
|
115
|
+
|
|
116
|
+
labelled_containers=""
|
|
117
|
+
if ! labelled_containers=$($SUDO docker ps --filter label=${BRAIN_WATCHDOG_LABEL_FILTER} --format '{{.ID}}' 2>/dev/null); then
|
|
118
|
+
echo "Warning: unable to verify running Brain containers with label ${BRAIN_WATCHDOG_LABEL_FILTER}." >&2
|
|
119
|
+
elif [ -z "$labelled_containers" ]; then
|
|
120
|
+
echo "Warning: no running Brain containers found with label ${BRAIN_WATCHDOG_LABEL_FILTER}." >&2
|
|
121
|
+
fi
|
|
122
|
+
`;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export async function installHealthWatchdog(): Promise<void> {
|
|
126
|
+
const serverIp = requireEnv("SERVER_IP");
|
|
127
|
+
const sshUser = requireEnv("SSH_USER").trim();
|
|
128
|
+
if (!sshUser) throw new Error("Missing SSH_USER");
|
|
129
|
+
const child = Bun.spawn(["ssh", `${sshUser}@${serverIp}`, "bash -s"], {
|
|
130
|
+
stdin: new Blob([buildHealthWatchdogInstallScript()]),
|
|
131
|
+
stdout: "inherit",
|
|
132
|
+
stderr: "inherit",
|
|
133
|
+
});
|
|
134
|
+
const exitCode = await child.exited;
|
|
135
|
+
if (exitCode !== 0) {
|
|
136
|
+
throw new Error(
|
|
137
|
+
`Health watchdog installation failed with exit ${exitCode}`,
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (import.meta.main) {
|
|
143
|
+
await installHealthWatchdog();
|
|
144
|
+
}
|
|
@@ -9,14 +9,23 @@ if (eventName === "workflow_dispatch") {
|
|
|
9
9
|
process.exit(0);
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
-
if (eventName !== "push") {
|
|
12
|
+
if (eventName !== "push" && eventName !== "workflow_run") {
|
|
13
13
|
throw new Error(`Unsupported GITHUB_EVENT_NAME: ${eventName}`);
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
const beforeSha = requireEnv("BEFORE_SHA");
|
|
17
|
-
const currentSha =
|
|
17
|
+
const currentSha =
|
|
18
|
+
eventName === "workflow_run"
|
|
19
|
+
? execFileSync("git", ["rev-parse", "HEAD"], {
|
|
20
|
+
encoding: "utf8",
|
|
21
|
+
}).trim()
|
|
22
|
+
: requireEnv("GITHUB_SHA");
|
|
18
23
|
|
|
19
|
-
if (
|
|
24
|
+
if (
|
|
25
|
+
!isUsableGitRevision(beforeSha) ||
|
|
26
|
+
!isUsableGitRevision(currentSha) ||
|
|
27
|
+
beforeSha === currentSha
|
|
28
|
+
) {
|
|
20
29
|
writeGitHubOutput("handles_json", JSON.stringify([]));
|
|
21
30
|
process.exit(0);
|
|
22
31
|
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import {
|
|
2
|
+
requireEnv,
|
|
3
|
+
runResolveMissingImages,
|
|
4
|
+
writeGitHubOutput,
|
|
5
|
+
} from "./helpers";
|
|
6
|
+
|
|
7
|
+
// The Build workflow's resolve step — the logic lives in @rizom/ops; this
|
|
8
|
+
// pilot repo only supplies its own paths and repository.
|
|
9
|
+
await runResolveMissingImages({
|
|
10
|
+
rootDir: process.cwd(),
|
|
11
|
+
imageRepository: `ghcr.io/${requireEnv("GITHUB_REPOSITORY")}`,
|
|
12
|
+
writeOutput: writeGitHubOutput,
|
|
13
|
+
});
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { derivePreviewDomain, loadPilotRegistry } from "@rizom/ops";
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
parseEnvFile,
|
|
7
|
+
requireEnv,
|
|
8
|
+
sitePackagesFor,
|
|
9
|
+
siteImageTag,
|
|
10
|
+
writeGitHubOutput,
|
|
11
|
+
} from "./helpers";
|
|
4
12
|
|
|
5
13
|
const handle = requireEnv("HANDLE");
|
|
6
14
|
const envPath = `users/${handle}/.env`;
|
|
@@ -17,23 +25,42 @@ if (!brainDomain) {
|
|
|
17
25
|
throw new Error(`Missing domain in ${brainYamlPath}`);
|
|
18
26
|
}
|
|
19
27
|
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
if (!zone) {
|
|
25
|
-
throw new Error(`Could not derive preview domain from ${brainDomain}`);
|
|
28
|
+
const registry = await loadPilotRegistry(process.cwd());
|
|
29
|
+
const user = registry.users.find((entry) => entry.handle === handle);
|
|
30
|
+
if (!user) {
|
|
31
|
+
throw new Error(`Unknown user handle: ${handle}`);
|
|
26
32
|
}
|
|
27
|
-
const previewDomain =
|
|
33
|
+
const previewDomain = derivePreviewDomain(brainDomain, {
|
|
34
|
+
sharedDomain: registry.pilot.domainSuffix,
|
|
35
|
+
});
|
|
36
|
+
const wwwDomain = isFleetDomain(
|
|
37
|
+
handle,
|
|
38
|
+
brainDomain,
|
|
39
|
+
registry.pilot.domainSuffix,
|
|
40
|
+
)
|
|
41
|
+
? ""
|
|
42
|
+
: `www.${brainDomain}`;
|
|
43
|
+
|
|
44
|
+
const brainVersion = envEntries["BRAIN_VERSION"] ?? "";
|
|
45
|
+
|
|
46
|
+
// The image tag is a pure function of this instance's own config: plain
|
|
47
|
+
// `brain-{version}` for a default instance, or its own `brain-{version}-sites-
|
|
48
|
+
// {hash}` when it declares a siteOverride. Resolved through the same helper the
|
|
49
|
+
// build uses so the tag we wait for and run matches exactly what was pushed.
|
|
50
|
+
const sitePackages = sitePackagesFor(user.siteOverride);
|
|
51
|
+
const imageTag = siteImageTag(brainVersion, sitePackages);
|
|
28
52
|
|
|
29
53
|
const outputs: Record<string, string> = {
|
|
30
|
-
brain_version:
|
|
54
|
+
brain_version: brainVersion,
|
|
31
55
|
content_repo: envEntries["CONTENT_REPO"] ?? "",
|
|
32
56
|
brain_domain: brainDomain,
|
|
33
57
|
preview_domain: previewDomain,
|
|
58
|
+
www_domain: wwwDomain,
|
|
59
|
+
cloudflare_zone_id: user.cloudflareZoneId ?? process.env["CF_ZONE_ID"] ?? "",
|
|
34
60
|
brain_yaml_path: brainYamlPath,
|
|
35
61
|
instance_name: `rover-${handle}`,
|
|
36
62
|
image_repository: `ghcr.io/${repository}`,
|
|
63
|
+
image_tag: imageTag,
|
|
37
64
|
registry_username: repositoryOwner,
|
|
38
65
|
};
|
|
39
66
|
|
|
@@ -47,3 +74,11 @@ for (const key of required) {
|
|
|
47
74
|
for (const [key, value] of Object.entries(outputs)) {
|
|
48
75
|
writeGitHubOutput(key, value);
|
|
49
76
|
}
|
|
77
|
+
|
|
78
|
+
function isFleetDomain(
|
|
79
|
+
userHandle: string,
|
|
80
|
+
domain: string,
|
|
81
|
+
pilotDomainSuffix: string,
|
|
82
|
+
): boolean {
|
|
83
|
+
return domain === `${userHandle}${pilotDomainSuffix}`;
|
|
84
|
+
}
|