@rizom/ops 0.2.0-alpha.2 → 0.2.0-alpha.200

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +9 -3
  2. package/dist/age-key-bootstrap.d.ts +17 -0
  3. package/dist/brains-ops.js +288 -173
  4. package/dist/cert-bootstrap.d.ts +24 -0
  5. package/dist/content-repo-ref.d.ts +10 -0
  6. package/dist/content-repo.d.ts +13 -0
  7. package/dist/default-user-runner.d.ts +1 -1
  8. package/dist/deploy.js +99 -166
  9. package/dist/entries/deploy.d.ts +3 -2
  10. package/dist/images.d.ts +76 -0
  11. package/dist/index.d.ts +7 -0
  12. package/dist/index.js +289 -173
  13. package/dist/load-registry.d.ts +38 -3
  14. package/dist/observed-status.d.ts +12 -0
  15. package/dist/onboard-user.d.ts +2 -2
  16. package/dist/origin-ca.d.ts +1 -0
  17. package/dist/parse-args.d.ts +4 -0
  18. package/dist/push-secrets.d.ts +2 -0
  19. package/dist/push-target.d.ts +1 -0
  20. package/dist/reconcile-all.d.ts +2 -2
  21. package/dist/reconcile-cohort.d.ts +2 -2
  22. package/dist/reconcile-lib.d.ts +4 -2
  23. package/dist/run-command.d.ts +8 -2
  24. package/dist/run-subprocess.d.ts +1 -0
  25. package/dist/schema.d.ts +68 -64
  26. package/dist/secrets-encrypt.d.ts +24 -0
  27. package/dist/secrets-push.d.ts +2 -5
  28. package/dist/ssh-key-bootstrap.d.ts +1 -0
  29. package/dist/user-add.d.ts +15 -0
  30. package/dist/user-runner.d.ts +5 -0
  31. package/dist/verify-user.d.ts +19 -0
  32. package/package.json +44 -40
  33. package/templates/rover-pilot/.env.schema +21 -2
  34. package/templates/rover-pilot/.github/workflows/build.yml +65 -17
  35. package/templates/rover-pilot/.github/workflows/deploy.yml +150 -45
  36. package/templates/rover-pilot/.github/workflows/reconcile.yml +22 -2
  37. package/templates/rover-pilot/README.md +8 -3
  38. package/templates/rover-pilot/deploy/scripts/decrypt-user-secrets.ts +119 -0
  39. package/templates/rover-pilot/deploy/scripts/helpers.ts +3 -0
  40. package/templates/rover-pilot/deploy/scripts/provision-server.ts +1 -1
  41. package/templates/rover-pilot/deploy/scripts/resolve-deploy-handles.ts +15 -4
  42. package/templates/rover-pilot/deploy/scripts/resolve-missing-images.ts +13 -0
  43. package/templates/rover-pilot/deploy/scripts/resolve-user-config.ts +68 -14
  44. package/templates/rover-pilot/deploy/scripts/sync-content-repo.ts +183 -0
  45. package/templates/rover-pilot/deploy/scripts/update-dns.ts +14 -4
  46. package/templates/rover-pilot/deploy/scripts/validate-secrets.ts +1 -1
  47. package/templates/rover-pilot/docs/onboarding-checklist.md +43 -8
  48. package/templates/rover-pilot/docs/operator-playbook.md +182 -0
  49. package/templates/rover-pilot/docs/user-onboarding.md +119 -0
  50. package/templates/rover-pilot/package.json +3 -0
  51. package/templates/rover-pilot/pilot.yaml +3 -0
  52. package/templates/rover-pilot/users/alice.yaml +5 -1
  53. package/dist/user-secret-names.d.ts +0 -6
  54. package/templates/rover-pilot/.kamal/hooks/pre-deploy +0 -9
  55. package/templates/rover-pilot/deploy/Dockerfile +0 -15
  56. package/templates/rover-pilot/deploy/kamal/deploy.yml +0 -39
@@ -9,14 +9,23 @@ if (eventName === "workflow_dispatch") {
9
9
  process.exit(0);
10
10
  }
11
11
 
12
- if (eventName !== "push") {
12
+ if (eventName !== "push" && eventName !== "workflow_run") {
13
13
  throw new Error(`Unsupported GITHUB_EVENT_NAME: ${eventName}`);
14
14
  }
15
15
 
16
16
  const beforeSha = requireEnv("BEFORE_SHA");
17
- const currentSha = requireEnv("GITHUB_SHA");
17
+ const currentSha =
18
+ eventName === "workflow_run"
19
+ ? execFileSync("git", ["rev-parse", "HEAD"], {
20
+ encoding: "utf8",
21
+ }).trim()
22
+ : requireEnv("GITHUB_SHA");
18
23
 
19
- if (!isUsableGitRevision(beforeSha) || !isUsableGitRevision(currentSha)) {
24
+ if (
25
+ !isUsableGitRevision(beforeSha) ||
26
+ !isUsableGitRevision(currentSha) ||
27
+ beforeSha === currentSha
28
+ ) {
20
29
  writeGitHubOutput("handles_json", JSON.stringify([]));
21
30
  process.exit(0);
22
31
  }
@@ -32,7 +41,9 @@ const handles = [
32
41
  diffOutput
33
42
  .split(/\r?\n/)
34
43
  .map((path) => {
35
- const match = path.match(/^users\/([^/]+)\/(?:\.env|brain\.yaml)$/);
44
+ const match =
45
+ path.match(/^users\/([^/]+)\/(?:\.env|brain\.yaml|content\/.*)$/) ??
46
+ path.match(/^users\/([^/]+)\.secrets\.yaml\.age$/);
36
47
  return match?.[1] ?? null;
37
48
  })
38
49
  .filter((handle): handle is string => handle !== null)
@@ -0,0 +1,13 @@
1
+ import {
2
+ requireEnv,
3
+ runResolveMissingImages,
4
+ writeGitHubOutput,
5
+ } from "./helpers";
6
+
7
+ // The Build workflow's resolve step — the logic lives in @rizom/ops; this
8
+ // pilot repo only supplies its own paths and repository.
9
+ await runResolveMissingImages({
10
+ rootDir: process.cwd(),
11
+ imageRepository: `ghcr.io/${requireEnv("GITHUB_REPOSITORY")}`,
12
+ writeOutput: writeGitHubOutput,
13
+ });
@@ -1,5 +1,14 @@
1
1
  import { readFileSync } from "node:fs";
2
- import { parseEnvFile, requireEnv, writeGitHubOutput } from "./helpers";
2
+
3
+ import { loadPilotRegistry } from "@rizom/ops";
4
+
5
+ import {
6
+ parseEnvFile,
7
+ requireEnv,
8
+ sitePackagesFor,
9
+ siteImageTag,
10
+ writeGitHubOutput,
11
+ } from "./helpers";
3
12
 
4
13
  const handle = requireEnv("HANDLE");
5
14
  const envPath = `users/${handle}/.env`;
@@ -12,32 +21,52 @@ const repositoryOwner = repository.split("/")[0] ?? "";
12
21
  const brainYaml = readFileSync(brainYamlPath, "utf8");
13
22
  const domainMatch = brainYaml.match(/^domain:\s*(.+)$/m);
14
23
  const brainDomain = domainMatch?.[1]?.trim().replace(/^['"]|['"]$/g, "") ?? "";
15
-
16
24
  if (!brainDomain) {
17
25
  throw new Error(`Missing domain in ${brainYamlPath}`);
18
26
  }
19
27
 
28
+ const registry = await loadPilotRegistry(process.cwd());
29
+ const user = registry.users.find((entry) => entry.handle === handle);
30
+ if (!user) {
31
+ throw new Error(`Unknown user handle: ${handle}`);
32
+ }
33
+ const previewDomain = resolvePreviewDomain(
34
+ handle,
35
+ brainDomain,
36
+ registry.pilot.domainSuffix,
37
+ );
38
+ const wwwDomain = isFleetDomain(
39
+ handle,
40
+ brainDomain,
41
+ registry.pilot.domainSuffix,
42
+ )
43
+ ? ""
44
+ : `www.${brainDomain}`;
45
+
46
+ const brainVersion = envEntries["BRAIN_VERSION"] ?? "";
47
+
48
+ // The image tag is a pure function of this instance's own config: plain
49
+ // `brain-{version}` for a default instance, or its own `brain-{version}-sites-
50
+ // {hash}` when it declares a siteOverride. Resolved through the same helper the
51
+ // build uses so the tag we wait for and run matches exactly what was pushed.
52
+ const sitePackages = sitePackagesFor(user.siteOverride);
53
+ const imageTag = siteImageTag(brainVersion, sitePackages);
54
+
20
55
  const outputs: Record<string, string> = {
21
- brain_version: envEntries["BRAIN_VERSION"] ?? "",
22
- ai_api_key_secret_name: envEntries["AI_API_KEY_SECRET"] ?? "",
23
- git_sync_token_secret_name: envEntries["GIT_SYNC_TOKEN_SECRET"] ?? "",
24
- mcp_auth_token_secret_name: envEntries["MCP_AUTH_TOKEN_SECRET"] ?? "",
25
- discord_bot_token_secret_name: envEntries["DISCORD_BOT_TOKEN_SECRET"] ?? "",
56
+ brain_version: brainVersion,
26
57
  content_repo: envEntries["CONTENT_REPO"] ?? "",
27
58
  brain_domain: brainDomain,
59
+ preview_domain: previewDomain,
60
+ www_domain: wwwDomain,
61
+ cloudflare_zone_id: user.cloudflareZoneId ?? process.env["CF_ZONE_ID"] ?? "",
28
62
  brain_yaml_path: brainYamlPath,
29
63
  instance_name: `rover-${handle}`,
30
64
  image_repository: `ghcr.io/${repository}`,
65
+ image_tag: imageTag,
31
66
  registry_username: repositoryOwner,
32
67
  };
33
68
 
34
- const required = [
35
- "brain_version",
36
- "ai_api_key_secret_name",
37
- "git_sync_token_secret_name",
38
- "mcp_auth_token_secret_name",
39
- "registry_username",
40
- ];
69
+ const required = ["brain_version", "registry_username"];
41
70
  for (const key of required) {
42
71
  if (!outputs[key]) {
43
72
  throw new Error(`Missing ${key} (derived from ${envPath})`);
@@ -47,3 +76,28 @@ for (const key of required) {
47
76
  for (const [key, value] of Object.entries(outputs)) {
48
77
  writeGitHubOutput(key, value);
49
78
  }
79
+
80
+ function resolvePreviewDomain(
81
+ userHandle: string,
82
+ domain: string,
83
+ pilotDomainSuffix: string,
84
+ ): string {
85
+ if (!isFleetDomain(userHandle, domain, pilotDomainSuffix)) {
86
+ return `preview.${domain}`;
87
+ }
88
+
89
+ const fleetZone = pilotDomainSuffix.replace(/^\./, "");
90
+ if (!fleetZone) {
91
+ throw new Error(`Could not derive preview domain from ${domain}`);
92
+ }
93
+
94
+ return `${userHandle}-preview.${fleetZone}`;
95
+ }
96
+
97
+ function isFleetDomain(
98
+ userHandle: string,
99
+ domain: string,
100
+ pilotDomainSuffix: string,
101
+ ): boolean {
102
+ return domain === `${userHandle}${pilotDomainSuffix}`;
103
+ }
@@ -0,0 +1,183 @@
1
+ import { cp, mkdtemp, mkdir, readFile, readdir } from "node:fs/promises";
2
+ import { existsSync } from "node:fs";
3
+ import { tmpdir } from "node:os";
4
+ import { dirname, join } from "node:path";
5
+ import { execFileSync } from "node:child_process";
6
+ import { readJsonResponse, requireEnv } from "./helpers";
7
+
8
+ const STALE_ANCHOR_PROFILE_MARKERS = [
9
+ "name: Your Name Here",
10
+ "Delete this and write your own",
11
+ ];
12
+
13
+ interface EnsureGitHubRepoOptions {
14
+ owner: string;
15
+ repo: string;
16
+ token: string;
17
+ }
18
+
19
+ interface GitHubRepoResponse {
20
+ clone_url?: string;
21
+ private?: boolean;
22
+ }
23
+
24
+ async function main(): Promise<void> {
25
+ const handle = requireEnv("HANDLE");
26
+ const contentRepo = requireEnv("CONTENT_REPO");
27
+ const token = requireEnv("GIT_SYNC_TOKEN");
28
+ const sourceDir = join("users", handle, "content");
29
+
30
+ if (!existsSync(sourceDir)) {
31
+ return;
32
+ }
33
+
34
+ const { owner, repo } = parseRepoSlug(contentRepo);
35
+ await ensureGitHubRepo({ owner, repo, token });
36
+
37
+ const tempRoot = await mkdtemp(join(tmpdir(), "brains-ops-content-"));
38
+ const checkoutDir = join(tempRoot, "repo");
39
+ const remoteUrl = buildAuthenticatedRemoteUrl(owner, repo, token);
40
+
41
+ runGit(["clone", remoteUrl, checkoutDir]);
42
+ runGit(["-C", checkoutDir, "checkout", "-B", "main"]);
43
+
44
+ const copiedFiles = await copyMissingFiles(sourceDir, checkoutDir);
45
+ if (copiedFiles === 0) {
46
+ return;
47
+ }
48
+
49
+ runGit(["-C", checkoutDir, "config", "user.name", "brains-ops[bot]"]);
50
+ runGit([
51
+ "-C",
52
+ checkoutDir,
53
+ "config",
54
+ "user.email",
55
+ "41898282+github-actions[bot]@users.noreply.github.com",
56
+ ]);
57
+ runGit(["-C", checkoutDir, "add", "."]);
58
+
59
+ if (hasNoStagedChanges(checkoutDir)) {
60
+ return;
61
+ }
62
+
63
+ runGit([
64
+ "-C",
65
+ checkoutDir,
66
+ "commit",
67
+ "-m",
68
+ `chore(content): seed ${handle} anchor profile`,
69
+ ]);
70
+ runGit(["-C", checkoutDir, "push", "origin", "HEAD:main"]);
71
+ }
72
+
73
+ async function ensureGitHubRepo(
74
+ options: EnsureGitHubRepoOptions,
75
+ ): Promise<void> {
76
+ const headers = {
77
+ Authorization: `Bearer ${options.token}`,
78
+ Accept: "application/vnd.github+json",
79
+ "Content-Type": "application/json",
80
+ };
81
+ const repoUrl = `https://api.github.com/repos/${options.owner}/${options.repo}`;
82
+ const repoResponse = await fetch(repoUrl, { headers });
83
+
84
+ if (repoResponse.ok) {
85
+ await readJsonResponse(repoResponse, "GitHub repo lookup");
86
+ return;
87
+ }
88
+
89
+ if (repoResponse.status !== 404) {
90
+ const payload = await readJsonResponse(repoResponse, "GitHub repo lookup");
91
+ throw new Error(`GitHub repo lookup failed: ${JSON.stringify(payload)}`);
92
+ }
93
+
94
+ const createResponse = await fetch(
95
+ `https://api.github.com/orgs/${options.owner}/repos`,
96
+ {
97
+ method: "POST",
98
+ headers,
99
+ body: JSON.stringify({
100
+ name: options.repo,
101
+ private: true,
102
+ auto_init: false,
103
+ }),
104
+ },
105
+ );
106
+ const payload = (await readJsonResponse(
107
+ createResponse,
108
+ "GitHub repo create",
109
+ )) as GitHubRepoResponse;
110
+
111
+ if (!createResponse.ok) {
112
+ throw new Error(`GitHub repo create failed: ${JSON.stringify(payload)}`);
113
+ }
114
+ }
115
+
116
+ function parseRepoSlug(contentRepo: string): { owner: string; repo: string } {
117
+ const [owner, repo] = contentRepo.split("/");
118
+ if (!owner || !repo) {
119
+ throw new Error(`Invalid CONTENT_REPO: ${contentRepo}`);
120
+ }
121
+ return { owner, repo };
122
+ }
123
+
124
+ function buildAuthenticatedRemoteUrl(
125
+ owner: string,
126
+ repo: string,
127
+ token: string,
128
+ ): string {
129
+ return `https://x-access-token:${encodeURIComponent(token)}@github.com/${owner}/${repo}.git`;
130
+ }
131
+
132
+ function runGit(args: string[]): void {
133
+ execFileSync("git", args, { stdio: "inherit" });
134
+ }
135
+
136
+ function hasNoStagedChanges(checkoutDir: string): boolean {
137
+ try {
138
+ execFileSync("git", ["-C", checkoutDir, "diff", "--cached", "--quiet"], {
139
+ stdio: "ignore",
140
+ });
141
+ return true;
142
+ } catch {
143
+ return false;
144
+ }
145
+ }
146
+
147
+ async function copyMissingFiles(
148
+ sourceDir: string,
149
+ targetDir: string,
150
+ ): Promise<number> {
151
+ const entries = await readdir(sourceDir, { withFileTypes: true });
152
+ let copiedFiles = 0;
153
+
154
+ for (const entry of entries) {
155
+ const sourcePath = join(sourceDir, entry.name);
156
+ const targetPath = join(targetDir, entry.name);
157
+
158
+ if (entry.isDirectory()) {
159
+ await mkdir(targetPath, { recursive: true });
160
+ copiedFiles += await copyMissingFiles(sourcePath, targetPath);
161
+ continue;
162
+ }
163
+
164
+ const existing = await readFile(targetPath, "utf8").catch(() => undefined);
165
+ if (existing !== undefined && !isStaleAnchorProfile(existing)) {
166
+ continue;
167
+ }
168
+
169
+ await mkdir(dirname(targetPath), { recursive: true });
170
+ await cp(sourcePath, targetPath, { force: true });
171
+ copiedFiles += existing === (await readFile(sourcePath, "utf8")) ? 0 : 1;
172
+ }
173
+
174
+ return copiedFiles;
175
+ }
176
+
177
+ function isStaleAnchorProfile(content: string): boolean {
178
+ return STALE_ANCHOR_PROFILE_MARKERS.some((marker) =>
179
+ content.includes(marker),
180
+ );
181
+ }
182
+
183
+ await main();
@@ -16,8 +16,11 @@ interface CloudflareResult {
16
16
  result?: Array<{ id: string }>;
17
17
  }
18
18
 
19
- async function upsertRecord(name: string): Promise<void> {
20
- const lookupUrl = `${baseUrl}/zones/${zoneId}/dns_records?type=A&name=${encodeURIComponent(name)}`;
19
+ async function findRecordId(
20
+ name: string,
21
+ type: "A" | "CNAME",
22
+ ): Promise<string | undefined> {
23
+ const lookupUrl = `${baseUrl}/zones/${zoneId}/dns_records?type=${type}&name=${encodeURIComponent(name)}`;
21
24
  const lookup = await fetch(lookupUrl, { headers });
22
25
  const payload = (await readJsonResponse(
23
26
  lookup,
@@ -27,9 +30,16 @@ async function upsertRecord(name: string): Promise<void> {
27
30
  throw new Error(`Cloudflare DNS lookup failed: ${JSON.stringify(payload)}`);
28
31
  }
29
32
 
30
- const existing = payload.result?.[0];
33
+ return payload.result?.[0]?.id;
34
+ }
35
+
36
+ async function upsertRecord(name: string): Promise<void> {
37
+ // Prefer an existing A record. If the hostname currently has a CNAME,
38
+ // replace that CNAME in-place so deploys can claim legacy www aliases.
39
+ const existing =
40
+ (await findRecordId(name, "A")) ?? (await findRecordId(name, "CNAME"));
31
41
  const url = existing
32
- ? `${baseUrl}/zones/${zoneId}/dns_records/${existing.id}`
42
+ ? `${baseUrl}/zones/${zoneId}/dns_records/${existing}`
33
43
  : `${baseUrl}/zones/${zoneId}/dns_records`;
34
44
 
35
45
  const response = await fetch(url, {
@@ -4,7 +4,7 @@ import { parseEnvSchema } from "./helpers";
4
4
  const envSchemaPath = ".env.schema";
5
5
  const schema = parseEnvSchema(readFileSync(envSchemaPath, "utf8"));
6
6
  const requiredKeys = schema
7
- .filter((entry) => entry.required)
7
+ .filter((entry) => entry.required && entry.key !== "BWS_ACCESS_TOKEN")
8
8
  .map((entry) => entry.key);
9
9
 
10
10
  const missing: string[] = [];
@@ -1,11 +1,46 @@
1
1
  # Onboarding Checklist
2
2
 
3
3
  1. Run `bun install` so the repo uses its pinned `@rizom/ops` version.
4
- 2. Fill in `pilot.yaml`.
5
- 3. Add or edit `users/<handle>.yaml`.
6
- 4. Add the user to a cohort in `cohorts/*.yaml`.
7
- 5. Run `bunx brains-ops render <repo>`.
8
- 6. Run `bunx brains-ops secrets:push <repo> <handle>`.
9
- 7. Run `bunx brains-ops onboard <repo> <handle>`.
10
- 8. For fleet upgrades, edit `pilot.yaml.brainVersion` and push once; CI rebuilds the shared image tag, refreshes generated user env files, and redeploys affected users.
11
- 9. Hand the MCP connection details to the user.
4
+ 2. Run `bunx brains-ops age-key:bootstrap <repo> --push-to gh`.
5
+ 3. Fill in `pilot.yaml`.
6
+ - keep your pinned `brainVersion`
7
+ - confirm shared selectors for `aiApiKey`, `gitSyncToken`, and `contentRepoAdminToken`
8
+ - use different tokens for `contentRepoAdminToken` and `gitSyncToken`: admin creates/checks content repos; sync is used by runtime directory-sync
9
+ - confirm `agePublicKey`
10
+ 4. Run `bunx brains-ops user:add <repo> <handle> --cohort <cohort>`.
11
+ - Web chat is the primary interface; it needs no per-user setup beyond the passkey.
12
+ - `user:add` currently writes `discord: enabled: true`; set it to `false` unless the user's cohort actually uses Discord.
13
+ - if the user should be an anchor on Discord, add `--anchor-id <discord-user-id>`.
14
+ - the command creates `users/<handle>.yaml`, `users/<handle>.secrets.yaml`, and the cohort membership without duplicating existing entries.
15
+ 5. Edit the generated user file if the anchor profile needs richer metadata.
16
+ - Set `setup.delivery: email` and `setup.email` so the user gets the passkey setup email — this is the default onboarding path.
17
+ - For ATProto publishing, add `atproto.identifier` to the user file; put only `atprotoAppPassword` in the per-user secrets file.
18
+ - Ensure `SETUP_EMAIL_API_KEY` and `SETUP_EMAIL_FROM` exist as GitHub Secrets before deploying any email-setup user.
19
+ 6. Run `bunx brains-ops render <repo>`.
20
+ 7. Run `bunx brains-ops ssh-key:bootstrap <repo> --push-to gh`.
21
+ 8. Run `bunx brains-ops cert:bootstrap <repo> --push-to gh`.
22
+ 9. Keep raw user secret material locally for now (`.env.local`, file-backed env vars, or equivalent local inputs), including `CONTENT_REPO_ADMIN_TOKEN` for operator onboarding.
23
+ 10. Run `bunx brains-ops secrets:encrypt <repo> <handle>`.
24
+ 11. Commit and push `users/<handle>.secrets.yaml.age`.
25
+ 12. Run `bunx brains-ops onboard <repo> <handle>`.
26
+ 13. Verify the deployed Rover contract:
27
+ - all presets:
28
+ - `https://<handle>.rizom.ai/health` returns `200`
29
+ - `https://<handle>.rizom.ai/chat` loads the web chat and accepts passkey sign-in
30
+ - `https://<handle>.rizom.ai/` loads the dashboard (or site surface on `default` preset)
31
+ - `https://<handle>.rizom.ai/cms` loads the CMS/login surface
32
+ - unauthenticated `POST https://<handle>.rizom.ai/mcp` returns the expected auth failure
33
+ - content repo exists and runtime sync is healthy
34
+ - background jobs are not repeatedly failing, except for expected missing optional integrations
35
+ - for `presetOverride: default` users:
36
+ - initial site build completes
37
+ 14. For fleet upgrades, edit `pilot.yaml.brainVersion` and push once; CI rebuilds the shared image tag, refreshes generated user env files, and redeploys affected users.
38
+ 15. Confirm the user received the setup email, registered their passkey, and can sign in to web chat at `https://<handle>.rizom.ai/chat`. That completes the default onboarding; everything below is per-cohort extras.
39
+ 16. Hand over the browser surfaces:
40
+ - Chat (primary): `https://<handle>.rizom.ai/chat`
41
+ - Dashboard: `https://<handle>.rizom.ai/`
42
+ - CMS: `https://<handle>.rizom.ai/cms`, plus GitHub token guidance if CMS editing is part of their cohort
43
+ 17. For Discord-enabled cohorts, hand the Discord setup details to the user as a secondary chat surface.
44
+ 18. If they need direct client access (MCP), use OAuth/passkey-capable clients where possible.
45
+ 19. If you are also giving them a content repo workflow, describe it as optional and frame git/Obsidian as an advanced file-based path, not the default.
46
+ 20. Send `docs/user-onboarding.md` to the user as the pilot handoff guide.
@@ -33,6 +33,36 @@ When a push changes only deploy contract files and no generated `users/<handle>/
33
33
 
34
34
  They are scaffolded from `@rizom/ops`, then versioned in this repo like any other deploy contract.
35
35
 
36
+ ## Stale deploy lock recovery
37
+
38
+ Kamal intentionally leaves its remote deploy lock in place when a deployment is cancelled or interrupted. Confirm that no deployment for the user is still active before releasing the lock, then use the deploy workflow's explicit recovery input:
39
+
40
+ ```sh
41
+ gh workflow run Deploy --ref main \
42
+ -f handle=<handle> \
43
+ -f release_stale_lock=true
44
+ ```
45
+
46
+ Recovery is opt-in and scoped to one handle. Normal push, reconcile, and manual deploy runs never remove a lock automatically.
47
+
48
+ ## Bootstrap flow
49
+
50
+ For this fleet, operator-local secret material remains the source of truth during onboarding and rotation. The repo stores encrypted per-user secrets, not raw values.
51
+
52
+ For a new pilot user, the operator bootstrap order is:
53
+
54
+ 1. `bunx brains-ops age-key:bootstrap <repo> --push-to gh`
55
+ 2. `bunx brains-ops ssh-key:bootstrap <repo> --push-to gh`
56
+ 3. `bunx brains-ops cert:bootstrap <repo> --push-to gh`
57
+ 4. `bunx brains-ops secrets:encrypt <repo> <handle>`
58
+ 5. `bunx brains-ops onboard <repo> <handle>`
59
+
60
+ `age-key:bootstrap` keeps a repo-local canonical age identity under `.brains-ops/age/identity.txt`, writes the matching public recipient to `pilot.yaml.agePublicKey`, and can push the private key to GitHub as `AGE_SECRET_KEY`.
61
+
62
+ The shared cert bootstrap writes local cert artifacts under `.brains-ops/certs/shared/`, which stays repo-local and ignored by git.
63
+
64
+ Preview hosts use the shape `<handle>-preview.rizom.ai`, so one wildcard origin cert for `*.rizom.ai` covers both the primary and preview hosts for every pilot user.
65
+
36
66
  ## Upgrading operator behavior
37
67
 
38
68
  When `@rizom/ops` changes the scaffolded deploy contract:
@@ -42,6 +72,158 @@ When `@rizom/ops` changes the scaffolded deploy contract:
42
72
  3. review the resulting changes to `.env.schema`, `deploy/scripts/`, and workflows in git
43
73
  4. commit the updated deploy artifacts together
44
74
 
75
+ ## Rover verification notes
76
+
77
+ Use the verification script after deploy:
78
+
79
+ ```sh
80
+ bunx brains-ops verify-user . <handle>
81
+ ```
82
+
83
+ It checks every Rover preset:
84
+
85
+ - `https://<handle>.rizom.ai/health` should return `200`
86
+ - unauthenticated `POST https://<handle>.rizom.ai/mcp` should return the expected auth failure
87
+ - background jobs should not be repeatedly failing, except for expected missing optional integrations
88
+
89
+ Additional `rover:core` note:
90
+
91
+ - Rover core is MCP-only; a bare `GET /` may return `401`, which does not indicate a bad deploy.
92
+
93
+ For `preset: default`, the script also checks:
94
+
95
+ - `https://<handle>.rizom.ai/` loads the browser/site surface
96
+ - `https://<handle>.rizom.ai/cms` loads the CMS/login surface
97
+
98
+ Manual checks that remain:
99
+
100
+ - initial site build is correct for the expected content/theme
101
+ - content repo exists and runtime sync is healthy beyond the basic `/health` response
102
+ - passkey setup/handoff is completed from the setup email
103
+
104
+ ## One-user `rover:default` baseline canary
105
+
106
+ Run this before adding custom site/theme packages or rolling a larger browser/CMS-first cohort.
107
+
108
+ 1. Create or choose a canary cohort with the default preset:
109
+
110
+ ```yaml
111
+ presetOverride: default
112
+ ```
113
+
114
+ 2. Add exactly one canary user to that cohort.
115
+ 3. For browser/CMS-first onboarding, configure setup email in `users/<handle>.yaml`:
116
+
117
+ ```yaml
118
+ setup:
119
+ delivery: email
120
+ email: user@example.com
121
+ ```
122
+
123
+ 4. Encrypt the user's secrets and commit only the `.age` file.
124
+ 5. Run `bunx brains-ops onboard . <handle>`.
125
+ 6. Run `bunx brains-ops verify-user . <handle>` with no custom site/theme overrides.
126
+ 7. Ask the user to complete passkey setup from the setup email.
127
+ 8. Continue to visual customization only after the canary is healthy.
128
+
129
+ Rollback:
130
+
131
+ - move the canary back to a core cohort, or remove `presetOverride: default` from the cohort
132
+ - reconcile generated outputs
133
+ - rebuild/redeploy the affected user
134
+
135
+ ## Setup email checklist
136
+
137
+ Use this for browser/CMS-first users who should receive their own first-passkey setup link by email.
138
+
139
+ 1. Add setup delivery to the user file:
140
+
141
+ ```yaml
142
+ setup:
143
+ delivery: email
144
+ email: user@example.com
145
+ ```
146
+
147
+ 2. Configure these GitHub Secrets before deploy:
148
+ - `SETUP_EMAIL_API_KEY`
149
+ - `SETUP_EMAIL_FROM`
150
+
151
+ 3. Reconcile/deploy the user or cohort:
152
+ - `bunx brains-ops onboard . <handle>`
153
+ - or `bunx brains-ops reconcile-cohort . <cohort>`
154
+
155
+ 4. Verify the generated `users/<handle>/brain.yaml` contains `auth-service.setupEmail` and `email-resend` config.
156
+ 5. Ask the user to complete passkey setup from the email link, then use:
157
+ - Dashboard: `https://<handle>.rizom.ai/`
158
+ - CMS: `https://<handle>.rizom.ai/cms`
159
+
160
+ Notes:
161
+
162
+ - The setup URL is generated and sent by the running brain; operators should not scrape logs or SSH into the instance to retrieve it.
163
+ - The auth service owns setup email dedupe. It should not resend for the same persisted setup token after restart, but should retry failed delivery and resend after token rotation.
164
+ - `SETUP_EMAIL_FROM` is not marked required because fleets without email setup can omit it, but it is required for users with `setup.delivery: email`.
165
+
166
+ ## AT Protocol smoke/config checklist
167
+
168
+ Use this when enabling AT Protocol publishing for a single pilot user.
169
+
170
+ 1. Add the public PDS identifier to the user file:
171
+
172
+ ```yaml
173
+ atproto:
174
+ identifier: rizom-test.bsky.social
175
+ ```
176
+
177
+ 2. Put the app password in `users/<handle>.secrets.yaml`:
178
+
179
+ ```yaml
180
+ atprotoAppPassword: <app-password>
181
+ ```
182
+
183
+ 3. Encrypt the per-user secret payload:
184
+ - `bunx brains-ops secrets:encrypt . <handle>`
185
+ 4. Reconcile/deploy the user or cohort:
186
+ - `bunx brains-ops onboard . <handle>`
187
+ - or `bunx brains-ops reconcile-cohort . <cohort>`
188
+ 5. Verify the generated `users/<handle>/brain.yaml` contains `plugins.atproto.identifier` and `appPassword: ${ATPROTO_APP_PASSWORD}`.
189
+
190
+ Notes:
191
+
192
+ - The ATProto identifier is public instance config and belongs in `users/<handle>.yaml`.
193
+ - The ATProto app password is secret and belongs only in the encrypted per-user secret payload.
194
+ - For smoke deployments, pin only the smoke cohort/user to the released brain version that contains ATProto support.
195
+
196
+ ## Discord bot token checklist
197
+
198
+ Use this when enabling Discord for a pilot user.
199
+
200
+ 1. Pick the user handle (for example `smoke`).
201
+ 2. Open the Discord Developer Portal.
202
+ 3. Create a **new application** for that user's rover.
203
+ 4. Add a **Bot** to the application.
204
+ 5. Copy the bot token.
205
+ 6. Put that value in `.env` or `.env.local` in this repo as `DISCORD_BOT_TOKEN=...` while onboarding that user.
206
+ 7. Keep `discord.enabled: true` in `users/<handle>.yaml` unless you explicitly want to disable the primary pilot interface.
207
+ 8. Encrypt the current per-user secret payload:
208
+ - `bunx brains-ops secrets:encrypt . <handle>`
209
+ 9. Reconcile/deploy the user or cohort:
210
+
211
+ - `bunx brains-ops onboard . <handle>`
212
+ - or `bunx brains-ops reconcile-cohort . <cohort>`
213
+
214
+ 11. In the Discord Developer Portal, generate an install URL and invite the bot to the right server.
215
+ 12. Send a test message in Discord and confirm the rover responds.
216
+
217
+ Notes:
218
+
219
+ - Use **one bot token per user/rover**.
220
+ - Do not reuse the same Discord bot token across multiple pilot users.
221
+ - Discord is the default pilot interface moving forward.
222
+ - The encrypted `users/<handle>.secrets.yaml.age` file is the durable checked-in deploy input; your local env is only the operator staging source.
223
+ - Direct MCP client access should use OAuth/passkey-capable clients where possible.
224
+ - When explaining the content workflow, describe it first as a normal **git repo** of **markdown/text files**.
225
+ - Position **Obsidian** as optional: it is just one possible editor for those same files, not the default requirement.
226
+
45
227
  ## Recovery notes
46
228
 
47
229
  Document known failure modes, recovery steps, and operator notes here.