@rizom/ops 0.2.0-alpha.21 → 0.2.0-alpha.210

package/templates/rover-pilot/deploy/scripts/resolve-missing-images.ts

@@ -0,0 +1,13 @@
+import {
+  requireEnv,
+  runResolveMissingImages,
+  writeGitHubOutput,
+} from "./helpers";
+
+// The Build workflow's resolve step — the logic lives in @rizom/ops; this
+// pilot repo only supplies its own paths and repository.
+await runResolveMissingImages({
+  rootDir: process.cwd(),
+  imageRepository: `ghcr.io/${requireEnv("GITHUB_REPOSITORY")}`,
+  writeOutput: writeGitHubOutput,
+});

package/templates/rover-pilot/deploy/scripts/resolve-user-config.ts

@@ -1,6 +1,14 @@
 import { readFileSync } from "node:fs";
 
-import { parseEnvFile, requireEnv, writeGitHubOutput } from "./helpers";
+import { loadPilotRegistry } from "@rizom/ops";
+
+import {
+  parseEnvFile,
+  requireEnv,
+  sitePackagesFor,
+  siteImageTag,
+  writeGitHubOutput,
+} from "./helpers";
 
 const handle = requireEnv("HANDLE");
 const envPath = `users/${handle}/.env`;
@@ -17,23 +25,44 @@ if (!brainDomain) {
   throw new Error(`Missing domain in ${brainYamlPath}`);
 }
 
-const zone =
-  brainDomain.startsWith(`${handle}.`) && brainDomain.length > handle.length + 1
-    ? brainDomain.slice(handle.length + 1)
-    : "";
-if (!zone) {
-  throw new Error(`Could not derive preview domain from ${brainDomain}`);
+const registry = await loadPilotRegistry(process.cwd());
+const user = registry.users.find((entry) => entry.handle === handle);
+if (!user) {
+  throw new Error(`Unknown user handle: ${handle}`);
 }
-const previewDomain = `${handle}-preview.${zone}`;
+const previewDomain = resolvePreviewDomain(
+  handle,
+  brainDomain,
+  registry.pilot.domainSuffix,
+);
+const wwwDomain = isFleetDomain(
+  handle,
+  brainDomain,
+  registry.pilot.domainSuffix,
+)
+  ? ""
+  : `www.${brainDomain}`;
+
+const brainVersion = envEntries["BRAIN_VERSION"] ?? "";
+
+// The image tag is a pure function of this instance's own config: plain
+// `brain-{version}` for a default instance, or its own `brain-{version}-sites-
+// {hash}` when it declares a siteOverride. Resolved through the same helper the
+// build uses so the tag we wait for and run matches exactly what was pushed.
+const sitePackages = sitePackagesFor(user.siteOverride);
+const imageTag = siteImageTag(brainVersion, sitePackages);
 
 const outputs: Record<string, string> = {
-  brain_version: envEntries["BRAIN_VERSION"] ?? "",
+  brain_version: brainVersion,
   content_repo: envEntries["CONTENT_REPO"] ?? "",
   brain_domain: brainDomain,
   preview_domain: previewDomain,
+  www_domain: wwwDomain,
+  cloudflare_zone_id: user.cloudflareZoneId ?? process.env["CF_ZONE_ID"] ?? "",
   brain_yaml_path: brainYamlPath,
   instance_name: `rover-${handle}`,
   image_repository: `ghcr.io/${repository}`,
+  image_tag: imageTag,
   registry_username: repositoryOwner,
 };
 
@@ -47,3 +76,28 @@ for (const key of required) {
 for (const [key, value] of Object.entries(outputs)) {
   writeGitHubOutput(key, value);
 }
+
+function resolvePreviewDomain(
+  userHandle: string,
+  domain: string,
+  pilotDomainSuffix: string,
+): string {
+  if (!isFleetDomain(userHandle, domain, pilotDomainSuffix)) {
+    return `preview.${domain}`;
+  }
+
+  const fleetZone = pilotDomainSuffix.replace(/^\./, "");
+  if (!fleetZone) {
+    throw new Error(`Could not derive preview domain from ${domain}`);
+  }
+
+  return `${userHandle}-preview.${fleetZone}`;
+}
+
+function isFleetDomain(
+  userHandle: string,
+  domain: string,
+  pilotDomainSuffix: string,
+): boolean {
+  return domain === `${userHandle}${pilotDomainSuffix}`;
+}

package/templates/rover-pilot/deploy/scripts/sync-content-repo.ts

@@ -5,53 +5,6 @@ import { dirname, join } from "node:path";
 import { execFileSync } from "node:child_process";
 import { readJsonResponse, requireEnv } from "./helpers";
 
-const handle = requireEnv("HANDLE");
-const contentRepo = requireEnv("CONTENT_REPO");
-const token = requireEnv("GIT_SYNC_TOKEN");
-const sourceDir = join("users", handle, "content");
-
-if (!existsSync(sourceDir)) {
-  process.exit(0);
-}
-
-const { owner, repo } = parseRepoSlug(contentRepo);
-await ensureGitHubRepo({ owner, repo, token });
-
-const tempRoot = await mkdtemp(join(tmpdir(), "brains-ops-content-"));
-const checkoutDir = join(tempRoot, "repo");
-const remoteUrl = buildAuthenticatedRemoteUrl(owner, repo, token);
-
-runGit(["clone", remoteUrl, checkoutDir]);
-runGit(["-C", checkoutDir, "checkout", "-B", "main"]);
-
-const copiedFiles = await copyMissingFiles(sourceDir, checkoutDir);
-if (copiedFiles === 0) {
-  process.exit(0);
-}
-
-runGit(["-C", checkoutDir, "config", "user.name", "brains-ops[bot]"]);
-runGit([
-  "-C",
-  checkoutDir,
-  "config",
-  "user.email",
-  "41898282+github-actions[bot]@users.noreply.github.com",
-]);
-runGit(["-C", checkoutDir, "add", "."]);
-
-if (hasNoStagedChanges(checkoutDir)) {
-  process.exit(0);
-}
-
-runGit([
-  "-C",
-  checkoutDir,
-  "commit",
-  "-m",
-  `chore(content): seed ${handle} anchor profile`,
-]);
-runGit(["-C", checkoutDir, "push", "origin", "HEAD:main"]);
-
 const STALE_ANCHOR_PROFILE_MARKERS = [
   "name: Your Name Here",
   "Delete this and write your own",
@@ -68,6 +21,55 @@ interface GitHubRepoResponse {
   private?: boolean;
 }
 
+async function main(): Promise<void> {
+  const handle = requireEnv("HANDLE");
+  const contentRepo = requireEnv("CONTENT_REPO");
+  const token = requireEnv("GIT_SYNC_TOKEN");
+  const sourceDir = join("users", handle, "content");
+
+  if (!existsSync(sourceDir)) {
+    return;
+  }
+
+  const { owner, repo } = parseRepoSlug(contentRepo);
+  await ensureGitHubRepo({ owner, repo, token });
+
+  const tempRoot = await mkdtemp(join(tmpdir(), "brains-ops-content-"));
+  const checkoutDir = join(tempRoot, "repo");
+  const remoteUrl = buildAuthenticatedRemoteUrl(owner, repo, token);
+
+  runGit(["clone", remoteUrl, checkoutDir]);
+  runGit(["-C", checkoutDir, "checkout", "-B", "main"]);
+
+  const copiedFiles = await copyMissingFiles(sourceDir, checkoutDir);
+  if (copiedFiles === 0) {
+    return;
+  }
+
+  runGit(["-C", checkoutDir, "config", "user.name", "brains-ops[bot]"]);
+  runGit([
+    "-C",
+    checkoutDir,
+    "config",
+    "user.email",
+    "41898282+github-actions[bot]@users.noreply.github.com",
+  ]);
+  runGit(["-C", checkoutDir, "add", "."]);
+
+  if (hasNoStagedChanges(checkoutDir)) {
+    return;
+  }
+
+  runGit([
+    "-C",
+    checkoutDir,
+    "commit",
+    "-m",
+    `chore(content): seed ${handle} anchor profile`,
+  ]);
+  runGit(["-C", checkoutDir, "push", "origin", "HEAD:main"]);
+}
+
 async function ensureGitHubRepo(
   options: EnsureGitHubRepoOptions,
 ): Promise<void> {
@@ -177,3 +179,5 @@ function isStaleAnchorProfile(content: string): boolean {
     content.includes(marker),
   );
 }
+
+await main();

package/templates/rover-pilot/deploy/scripts/update-dns.ts

@@ -16,8 +16,11 @@ interface CloudflareResult {
   result?: Array<{ id: string }>;
 }
 
-async function upsertRecord(name: string): Promise<void> {
-  const lookupUrl = `${baseUrl}/zones/${zoneId}/dns_records?type=A&name=${encodeURIComponent(name)}`;
+async function findRecordId(
+  name: string,
+  type: "A" | "CNAME",
+): Promise<string | undefined> {
+  const lookupUrl = `${baseUrl}/zones/${zoneId}/dns_records?type=${type}&name=${encodeURIComponent(name)}`;
   const lookup = await fetch(lookupUrl, { headers });
   const payload = (await readJsonResponse(
     lookup,
@@ -27,9 +30,16 @@ async function upsertRecord(name: string): Promise<void> {
     throw new Error(`Cloudflare DNS lookup failed: ${JSON.stringify(payload)}`);
   }
 
-  const existing = payload.result?.[0];
+  return payload.result?.[0]?.id;
+}
+
+async function upsertRecord(name: string): Promise<void> {
+  // Prefer an existing A record. If the hostname currently has a CNAME,
+  // replace that CNAME in-place so deploys can claim legacy www aliases.
+  const existing =
+    (await findRecordId(name, "A")) ?? (await findRecordId(name, "CNAME"));
   const url = existing
-    ? `${baseUrl}/zones/${zoneId}/dns_records/${existing.id}`
+    ? `${baseUrl}/zones/${zoneId}/dns_records/${existing}`
     : `${baseUrl}/zones/${zoneId}/dns_records`;
 
   const response = await fetch(url, {

package/templates/rover-pilot/deploy/scripts/validate-secrets.ts

@@ -4,7 +4,7 @@ import { parseEnvSchema } from "./helpers";
 const envSchemaPath = ".env.schema";
 const schema = parseEnvSchema(readFileSync(envSchemaPath, "utf8"));
 const requiredKeys = schema
-  .filter((entry) => entry.required)
+  .filter((entry) => entry.required && entry.key !== "BWS_ACCESS_TOKEN")
   .map((entry) => entry.key);
 
 const missing: string[] = [];

package/templates/rover-pilot/docs/onboarding-checklist.md

@@ -4,23 +4,43 @@
 2. Run `bunx brains-ops age-key:bootstrap <repo> --push-to gh`.
 3. Fill in `pilot.yaml`.
    - keep your pinned `brainVersion`
-   - confirm shared selectors for `aiApiKey`, `gitSyncToken`, and `mcpAuthToken`
+   - confirm shared selectors for `aiApiKey`, `gitSyncToken`, and `contentRepoAdminToken`
+   - use different tokens for `contentRepoAdminToken` and `gitSyncToken`: admin creates/checks content repos; sync is used by runtime directory-sync
    - confirm `agePublicKey`
-4. Add or edit `users/<handle>.yaml`.
-   - Discord is enabled by default for pilot users
-   - if the user should be an anchor there, set `discord.anchorUserId` to their Discord user ID
-5. Add the user to a cohort in `cohorts/*.yaml`.
+4. Run `bunx brains-ops user:add <repo> <handle> --cohort <cohort>`.
+   - Web chat is the primary interface; it needs no per-user setup beyond the passkey.
+   - `user:add` currently writes `discord: enabled: true`; set it to `false` unless the user's cohort actually uses Discord.
+   - if the user should be an anchor on Discord, add `--anchor-id <discord-user-id>`.
+   - the command creates `users/<handle>.yaml`, `users/<handle>.secrets.yaml`, and the cohort membership without duplicating existing entries.
+5. Edit the generated user file if the anchor profile needs richer metadata.
+   - Set `setup.delivery: email` and `setup.email` so the user gets the passkey setup email — this is the default onboarding path.
+   - For ATProto publishing, add `atproto.identifier` to the user file; put only `atprotoAppPassword` in the per-user secrets file.
+   - Ensure `SETUP_EMAIL_API_KEY` and `SETUP_EMAIL_FROM` exist as GitHub Secrets before deploying any email-setup user.
 6. Run `bunx brains-ops render <repo>`.
 7. Run `bunx brains-ops ssh-key:bootstrap <repo> --push-to gh`.
 8. Run `bunx brains-ops cert:bootstrap <repo> --push-to gh`.
-9. Keep raw user secret material locally for now (`.env.local`, file-backed env vars, or equivalent local inputs).
+9. Keep raw user secret material locally for now (`.env.local`, file-backed env vars, or equivalent local inputs), including `CONTENT_REPO_ADMIN_TOKEN` for operator onboarding.
 10. Run `bunx brains-ops secrets:encrypt <repo> <handle>`.
 11. Commit and push `users/<handle>.secrets.yaml.age`.
 12. Run `bunx brains-ops onboard <repo> <handle>`.
-13. Verify the deployed rover core contract:
-    - `https://<handle>.rizom.ai/health` returns `200`
-    - unauthenticated `POST https://<handle>.rizom.ai/mcp` returns `401`
+13. Verify the deployed Rover contract:
+    - all presets:
+      - `https://<handle>.rizom.ai/health` returns `200`
+      - `https://<handle>.rizom.ai/chat` loads the web chat and accepts passkey sign-in
+      - `https://<handle>.rizom.ai/` loads the dashboard (or site surface on `default` preset)
+      - `https://<handle>.rizom.ai/cms` loads the CMS/login surface
+      - unauthenticated `POST https://<handle>.rizom.ai/mcp` returns the expected auth failure
+      - content repo exists and runtime sync is healthy
+      - background jobs are not repeatedly failing, except for expected missing optional integrations
+    - for `presetOverride: default` users:
+      - initial site build completes
 14. For fleet upgrades, edit `pilot.yaml.brainVersion` and push once; CI rebuilds the shared image tag, refreshes generated user env files, and redeploys affected users.
-15. Hand the Discord setup details to the user. If they need direct client access, also hand over the MCP connection details.
-16. If you are also giving them a content repo workflow, describe it first as a normal git repo of markdown/text files; mention Obsidian only as an optional editor.
-17. Send `docs/user-onboarding.md` to the user as the pilot handoff guide.
+15. Confirm the user received the setup email, registered their passkey, and can sign in to web chat at `https://<handle>.rizom.ai/chat`. That completes the default onboarding; everything below is per-cohort extras.
+16. Hand over the browser surfaces:
+    - Chat (primary): `https://<handle>.rizom.ai/chat`
+    - Dashboard: `https://<handle>.rizom.ai/`
+    - CMS: `https://<handle>.rizom.ai/cms`, plus GitHub token guidance if CMS editing is part of their cohort
+17. For Discord-enabled cohorts, hand the Discord setup details to the user as a secondary chat surface.
+18. If they need direct client access (MCP), use OAuth/passkey-capable clients where possible.
+19. If you are also giving them a content repo workflow, describe it as optional and frame git/Obsidian as an advanced file-based path, not the default.
+20. Send `docs/user-onboarding.md` to the user as the pilot handoff guide.

package/templates/rover-pilot/docs/operator-playbook.md

@@ -33,6 +33,18 @@ When a push changes only deploy contract files and no generated `users/<handle>/
 
 They are scaffolded from `@rizom/ops`, then versioned in this repo like any other deploy contract.
 
+## Stale deploy lock recovery
+
+Kamal intentionally leaves its remote deploy lock in place when a deployment is cancelled or interrupted. Confirm that no deployment for the user is still active before releasing the lock, then use the deploy workflow's explicit recovery input:
+
+```sh
+gh workflow run Deploy --ref main \
+  -f handle=<handle> \
+  -f release_stale_lock=true
+```
+
+Recovery is opt-in and scoped to one handle. Normal push, reconcile, and manual deploy runs never remove a lock automatically.
+
 ## Bootstrap flow
 
 For this fleet, operator-local secret material remains the source of truth during onboarding and rotation. The repo stores encrypted per-user secrets, not raw values.
@@ -60,15 +72,136 @@ When `@rizom/ops` changes the scaffolded deploy contract:
 3. review the resulting changes to `.env.schema`, `deploy/scripts/`, and workflows in git
 4. commit the updated deploy artifacts together
 
-## Rover-core verification notes
+## Rover verification notes
+
+Use the verification script after deploy:
 
-Rover core is MCP-only. Do not expect the bare domain to serve a website.
+```sh
+bunx brains-ops verify-user . <handle>
+```
 
-Use these checks after deploy:
+It checks every Rover preset:
 
 - `https://<handle>.rizom.ai/health` should return `200`
-- unauthenticated `POST https://<handle>.rizom.ai/mcp` should return `401 Unauthorized: Bearer token required`
-- a bare `GET /` may also return `401`; that is expected for rover core and does not indicate a bad deploy
+- unauthenticated `POST https://<handle>.rizom.ai/mcp` should return the expected auth failure
+- background jobs should not be repeatedly failing, except for expected missing optional integrations
+
+Additional `rover:core` note:
+
+- Rover core is MCP-only; a bare `GET /` may return `401`, which does not indicate a bad deploy.
+
+For `preset: default`, the script also checks:
+
+- `https://<handle>.rizom.ai/` loads the browser/site surface
+- `https://<handle>.rizom.ai/cms` loads the CMS/login surface
+
+Manual checks that remain:
+
+- initial site build is correct for the expected content/theme
+- content repo exists and runtime sync is healthy beyond the basic `/health` response
+- passkey setup/handoff is completed from the setup email
+
+## One-user `rover:default` baseline canary
+
+Run this before adding custom site/theme packages or rolling a larger browser/CMS-first cohort.
+
+1. Create or choose a canary cohort with the default preset:
+
+   ```yaml
+   presetOverride: default
+   ```
+
+2. Add exactly one canary user to that cohort.
+3. For browser/CMS-first onboarding, configure setup email in `users/<handle>.yaml`:
+
+   ```yaml
+   setup:
+     delivery: email
+     email: user@example.com
+   ```
+
+4. Encrypt the user's secrets and commit only the `.age` file.
+5. Run `bunx brains-ops onboard . <handle>`.
+6. Run `bunx brains-ops verify-user . <handle>` with no custom site/theme overrides.
+7. Ask the user to complete passkey setup from the setup email.
+8. Continue to visual customization only after the canary is healthy.
+
+Rollback:
+
+- move the canary back to a core cohort, or remove `presetOverride: default` from the cohort
+- reconcile generated outputs
+- rebuild/redeploy the affected user
+
+## Setup email checklist
+
+Use this for browser/CMS-first users who should receive their own first-passkey setup link by email.
+
+1. Add setup delivery to the user file:
+
+   ```yaml
+   setup:
+     delivery: email
+     email: user@example.com
+   ```
+
+2. Configure these GitHub Secrets before deploy:
+   - `SETUP_EMAIL_API_KEY`
+   - `SETUP_EMAIL_FROM`
+
+3. Reconcile/deploy the user or cohort:
+   - `bunx brains-ops onboard . <handle>`
+   - or `bunx brains-ops reconcile-cohort . <cohort>`
+
+4. Verify the generated `users/<handle>/brain.yaml` contains `auth-service.setupEmail` and `email-resend` config.
+5. Ask the user to complete passkey setup from the email link, then use:
+   - Dashboard: `https://<handle>.rizom.ai/`
+   - CMS: `https://<handle>.rizom.ai/cms`
+
+Notes:
+
+- The setup URL is generated and sent by the running brain; operators should not scrape logs or SSH into the instance to retrieve it.
+- The auth service owns setup email dedupe. It should not resend for the same persisted setup token after restart, but should retry failed delivery and resend after token rotation.
+- `SETUP_EMAIL_FROM` is not marked required because fleets without email setup can omit it, but it is required for users with `setup.delivery: email`.
+
+## AT Protocol smoke/config checklist
+
+Use this when enabling AT Protocol publishing for a single pilot user.
+
+1. Add the public PDS identifier to the user file. Prefer the account DID as
+   the identifier — it survives handle changes. Add `accountDid` too when the
+   member wants their handle verified against their subdomain
+   (`@<handle>.<domainSuffix>`): the brain then serves it at
+   `/.well-known/atproto-did` and Bluesky's "I have my own domain" HTTP
+   verification passes with no DNS records.
+
+   ```yaml
+   atproto:
+     identifier: did:plc:example123
+     accountDid: did:plc:example123
+   ```
+
+   Only for the PDS account designated by the protocol authority's `_lexicon`
+   DNS TXT record, also set `lexiconAuthority: true`. Every other fleet user
+   must omit it.
+
+2. Put the app password in `users/<handle>.secrets.yaml`:
+
+   ```yaml
+   atprotoAppPassword: <app-password>
+   ```
+
+3. Encrypt the per-user secret payload:
+   - `bunx brains-ops secrets:encrypt . <handle>`
+4. Reconcile/deploy the user or cohort:
+   - `bunx brains-ops onboard . <handle>`
+   - or `bunx brains-ops reconcile-cohort . <cohort>`
+5. Verify the generated `users/<handle>/brain.yaml` contains `plugins.atproto.identifier` (plus `accountDid` and `lexiconAuthority` when configured) and `appPassword: ${ATPROTO_APP_PASSWORD}`.
+
+Notes:
+
+- The ATProto identifier and authority flag are public instance config and belong in `users/<handle>.yaml`. Only the DNS-designated authority account may set `lexiconAuthority: true`.
+- The ATProto app password is secret and belongs only in the encrypted per-user secret payload.
+- For smoke deployments, pin only the smoke cohort/user to the released brain version that contains ATProto support.
 
 ## Discord bot token checklist
 
@@ -97,7 +230,7 @@ Notes:
 - Do not reuse the same Discord bot token across multiple pilot users.
 - Discord is the default pilot interface moving forward.
 - The encrypted `users/<handle>.secrets.yaml.age` file is the durable checked-in deploy input; your local env is only the operator staging source.
-- MCP is optional and mainly for direct client access or specific testing workflows.
+- Direct MCP client access should use OAuth/passkey-capable clients where possible.
 - When explaining the content workflow, describe it first as a normal **git repo** of **markdown/text files**.
 - Position **Obsidian** as optional: it is just one possible editor for those same files, not the default requirement.