@openmeet/snowglobe 0.0.0-stage → 0.3.0-rc.1

package/lib/commands/render.mjs

@@ -0,0 +1,63 @@
+// bin/snowglobe.mjs render [<file>] [--out <dir>] [--domain <domain>]
+//
+// render checks the file as check does (exit 1 with one line per problem), then writes the https
+// globe it describes into <dir> (globe/ in the snowglobe's state folder by default, see
+// lib/state.mjs): compose.yml, nginx's servers, the leaf certificate's names and globe.env, the
+// facts the other commands read (lib/render.mjs). Nothing in them is secret. --domain replaces the file's domain, for a copy beside another globe. A group
+// host with no image or build folder in the file is built from the folder its kind's variable
+// names (SG_OPENSOCIAL_BUILD, a checkout of atproto-group-host); with neither, render fails.
+
+import { mkdirSync, renameSync, writeFileSync } from 'node:fs';
+import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
+import { withGroupHostSources } from '../config.mjs';
+import { ROOT, paths } from '../env.mjs';
+import { loadBase, renderGlobe } from '../render.mjs';
+import { claimState, session } from '../state.mjs';
+import { checkedGlobe } from './check.mjs';
+
+export const command = {
+  name: 'render',
+  usage: 'render [<file>] [--out <dir>] [--domain <domain>]',
+  options: { out: { type: 'string' }, domain: { type: 'string' } },
+  positionals: [0, 1],
+};
+
+export function run({ values, positionals }) {
+  const { path, globe } = checkedGlobe(positionals[0], { domain: values.domain });
+  if (!globe) return 1;
+  let out;
+  try {
+    // Into the state folder, the file takes it over; into --out, it only reads where the state is.
+    const s = session({ file: path });
+    if (values.out === undefined) claimState(s);
+    out = writeGlobe(globe, path, values.out ?? paths(ROOT, s.state).globe, process.env, s.state);
+  } catch (e) {
+    for (const line of e.message.split('\n')) console.error(`${path}: ${line}`);
+    return 1;
+  }
+  console.log(`${path}: wrote ${out}: globe ${globe.globe} at ${globe.domain}, PDSs ${globe.pds.map((p) => p.name).join(' ')}`);
+  return 0;
+}
+
+/**
+ * Write the checked `globe`, read from `path`, into `out`, a folder; returns it. `state` is the
+ * snowglobe's state folder, which the compose file mounts from. Throws, before writing anything,
+ * when a group host has nowhere to get its image from.
+ */
+export function writeGlobe(globe, path, out, env, state) {
+  globe = withGroupHostSources(globe, env);
+  // The compose file mounts nginx's part of the output, by a path from the repo root, which is
+  // the compose project directory (lib/env.mjs dc).
+  out = resolve(out);
+  const rel = relative(ROOT, out);
+  const globeDir = rel && !rel.startsWith('..') && !isAbsolute(rel) ? rel : out;
+  const files = renderGlobe(globe, loadBase(ROOT), { stateDir: state, globeDir, source: basename(path) });
+  for (const [name, text] of Object.entries(files)) {
+    const target = join(out, name);
+    mkdirSync(dirname(target), { recursive: true });
+    // Written aside and renamed, so nothing reads half a file.
+    writeFileSync(`${target}.tmp`, text);
+    renameSync(`${target}.tmp`, target);
+  }
+  return out;
+}

package/lib/commands/run.mjs

@@ -0,0 +1,109 @@
+// bin/snowglobe.mjs run [--app <name>] [--] <cmd> [args...]
+//
+// run runs a command with the snowglobe around it, for that command's process tree only:
+//   - it is handed the app's settings, the apps[].env of the snowglobe.yml (lib/commands/env.mjs;
+//     the same as snowglobe env, secrets included, none printed). They win over a value already in
+//     the environment, and one line names the ones they replaced. --app picks the app when the
+//     file lists several;
+//   - the globe's names (its page, each PDS, its services and plc.directory, from the rendered
+//     globe), every handle in https/names (account and up add them) and every app in
+//     https/apps, as <name>.app.<domain>, resolve to SG_BIND, where nginx answers on
+//     443 (lib/hosts.mjs). HTTPS_RUN_EXTRA_NAMES adds names (space-separated);
+//   - Node trusts the globe's CA (NODE_EXTRA_CA_CERTS=<state root>/ca/ca.crt), on top of its own
+//     roots. Miniflare passes the same file on to workerd.
+//
+// The globe is the one the snowglobe.yml in this folder (or above) describes, as for up.
+//
+// e.g. bin/snowglobe.mjs run node scripts/https-probe.mjs
+//      (cd ../myapp && ../snowglobe/bin/snowglobe.mjs run npm run dev)
+//
+// How: bubblewrap gives the command its own mount namespace with a generated hosts file bound
+// over /etc/hosts. Nothing else changes: same filesystem, same network, same user. Outside,
+// every name resolves as usual. The names are read when the command starts, so a handle or app
+// added later needs a restart of the command. Linux only: a Mac has no bubblewrap, and reaches
+// the globe through snowglobe env instead (the names resolve through public DNS, and the CA is in
+// NODE_EXTRA_CA_CERTS).
+
+import { spawn } from 'node:child_process';
+import { accessSync, constants, existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { constants as os } from 'node:os';
+import { delimiter, join } from 'node:path';
+import { readApps } from '../apps.mjs';
+import { ROOT, paths, resolveBind, resolveDomain } from '../env.mjs';
+import { hostsFile, runNames } from '../hosts.mjs';
+import { session } from '../state.mjs';
+import { appSettings } from './env.mjs';
+
+export const command = {
+  name: 'run',
+  usage: 'run [--app <name>] [--] <cmd> [args...]',
+  // Every word after run is the command's own, dashes and all, except an --app <name> first of
+  // all; none is a usage mistake.
+  parse: (args) => {
+    const named = args[0] === '--app';
+    if (named && args.length < 2) return null;
+    const rest = named ? args.slice(2) : args;
+    const cmd = rest[0] === '--' ? rest.slice(1) : rest;
+    return cmd.length ? { cmd, app: named ? args[1] : undefined } : null;
+  },
+};
+
+// The first `name` on PATH that can be run, or null.
+function onPath(name, env) {
+  for (const dir of (env.PATH ?? '').split(delimiter).filter(Boolean)) {
+    try {
+      accessSync(join(dir, name), constants.X_OK);
+      return join(dir, name);
+    } catch {}
+  }
+  return null;
+}
+
+export const run = ({ cmd, app }) => runUnder(cmd, { settings: appSettings({ app, secrets: true }).vars });
+
+/**
+ * Run `cmd` (a word list) under the globe's names with `settings` ([{ name, value }], from
+ * appSettings) in its environment; resolves to its exit code. A setting replaces a value of the same
+ * name already in `env`, and `log` is told which.
+ */
+export function runUnder(cmd, { root = ROOT, env = process.env, state = session({ env }).state, settings = [], log = (m) => console.error(m) } = {}) {
+  if (process.platform !== 'linux')
+    throw new Error('run needs Linux (bubblewrap) for now; on this system eval "$(snowglobe env)" hands the app the same settings');
+  const p = paths(root, state);
+  if (!existsSync(p.ca)) throw new Error(`no ${p.ca}; run snowglobe up (or certs) first`);
+  if (!existsSync(p.leafNames)) throw new Error(`no ${p.leafNames}; run snowglobe up first`);
+  const bwrap = onPath('bwrap', env);
+  if (!bwrap) throw new Error('bwrap (bubblewrap) is not installed, or not on PATH');
+  // Refuses a SG_DOMAIN the render is not for, and a SG_BIND that is not loopback.
+  const { domain } = resolveDomain({ root, state, env });
+  const bind = resolveBind({ root, state, env });
+
+  const read = (f) => (existsSync(f) ? readFileSync(f, 'utf8') : '');
+  const names = runNames({ leafNames: read(p.leafNames), handles: read(p.names), apps: readApps(p), domain, extra: env.HTTPS_RUN_EXTRA_NAMES });
+  // Written aside and renamed, so a run already holding the old file is undisturbed.
+  const hosts = join(p.https, 'hosts');
+  writeFileSync(`${hosts}.tmp.${process.pid}`, hostsFile(read('/etc/hosts'), names, bind), { mode: 0o644 });
+  renameSync(`${hosts}.tmp.${process.pid}`, hosts);
+
+  // Never the values: a setting may be a login.
+  const replaced = settings.filter((x) => !x.auto && env[x.name] !== undefined && env[x.name] !== x.value).map((x) => x.name);
+  if (replaced.length) log(`snowglobe run: the snowglobe's settings replaced ${replaced.join(', ')} from the environment`);
+
+  const child = spawn(bwrap, ['--dev-bind', '/', '/', '--ro-bind', hosts, '/etc/hosts', '--die-with-parent', '--', ...cmd], {
+    stdio: 'inherit',
+    env: { ...env, NODE_EXTRA_CA_CERTS: p.ca, ...Object.fromEntries(settings.map((x) => [x.name, x.value])) },
+  });
+  // A terminal's ^C reaches the whole process group, the command included, so it is left to the
+  // command; a signal sent to run alone is passed on.
+  process.on('SIGINT', () => {});
+  for (const sig of ['SIGTERM', 'SIGHUP']) process.on(sig, () => child.kill(sig));
+  return new Promise((done, fail) => {
+    // A spawn error can quote an environment value; a setting may be a login, so it is named instead.
+    child.on('error', (e) => {
+      let msg = e.message;
+      for (const x of settings) if (x.value) msg = msg.split(x.value).join(`<${x.name}>`);
+      fail(Object.assign(new Error(msg), { code: e.code }));
+    });
+    child.on('exit', (code, signal) => done(signal ? 128 + (os.signals[signal] ?? 0) : code));
+  });
+}

package/lib/commands/trust.mjs

@@ -0,0 +1,45 @@
+// bin/snowglobe.mjs trust [--remove] [--dry-run] [--no-sudo]
+//
+// trust puts your snowglobe CA (one per user, <state root>/ca/ca.crt, made by certs or up) where a
+// browser or program looks for the CAs it trusts, so https://local.snowglobe.at opens without a
+// warning: the system store and the NSS databases Chrome, Edge and Firefox use on Linux; on WSL
+// also the Windows store, whose command it prints for PowerShell; the login keychain on macOS; the
+// user store on Windows (lib/trust.mjs). Every command is printed as it runs, and the ones that
+// need sudo are marked. Running it again adds nothing. --remove takes the CA out of all of them.
+// --dry-run prints what it would run and changes nothing. --no-sudo skips the steps that need sudo
+// and prints them for you. The CA is limited to the snowglobe's names (lib/certs.mjs), so trusting
+// it does not let it vouch for other sites.
+//
+// Exit 0 when every step is done or in place, 1 when one could not be (a tool is missing, a command
+// failed, sudo skipped), 2 when there is no CA yet. Steps left for you to run (Windows from WSL,
+// Firefox on macOS) do not change the exit code.
+
+import { existsSync } from 'node:fs';
+import { stateRoot } from '../env.mjs';
+import { join } from 'node:path';
+
+export const command = {
+  name: 'trust',
+  usage: 'trust [--remove] [--dry-run] [--no-sudo]',
+  options: { remove: { type: 'boolean' }, 'dry-run': { type: 'boolean' }, 'no-sudo': { type: 'boolean' } },
+  positionals: [0, 0],
+  // Each flag once at most.
+  accepts: ({ tokens }) => new Set(tokens.filter((t) => t.kind === 'option').map((t) => t.name)).size === tokens.length,
+};
+
+export async function run({ values }, { ctx } = {}) {
+  const { applySteps, realContext, trustSteps } = await import('../trust.mjs');
+  const ca = join(stateRoot(), 'ca', 'ca.crt');
+  if (!existsSync(ca)) {
+    console.error(`snowglobe trust: no CA at ${ca}. Run snowglobe up (or snowglobe certs) first, which makes it`);
+    return 2;
+  }
+  const c = ctx ?? realContext();
+  const remove = Boolean(values.remove);
+  console.log(`${remove ? 'Removing' : 'Trusting'} ${ca}${values['dry-run'] ? ' (dry run: nothing is changed)' : ''}`);
+  const r = applySteps(trustSteps(ca, c), { remove, dryRun: Boolean(values['dry-run']), noSudo: Boolean(values['no-sudo']), ctx: c });
+  console.log(r.failed ? `${r.failed} step(s) not done.` : 'Done.');
+  if (r.manual) console.log(`${r.manual} step(s) are for you to run, as printed above.`);
+  if (!remove && !r.failed) console.log('Restart a browser that is open for it to see the CA.');
+  return r.failed ? 1 : 0;
+}

package/lib/commands/up.mjs

@@ -0,0 +1,301 @@
+// bin/snowglobe.mjs up [<file> | <name>] [-f <file>] [--default]
+//
+// up brings up the globe a snowglobe.yml describes, as the compose project named for the file's
+// `globe:` (or SG_PROJECT). Safe to rerun: each step keeps what an earlier run made, and a rerun
+// is one `docker compose up`.
+//
+// Everything it makes is kept in the snowglobe's state folder, <state root>/<globe> (lib/env.mjs
+// stateRoot: ~/.local/share/snowglobe, or under XDG_DATA_HOME; %LOCALAPPDATA% on Windows), which
+// remembers the file that owns it and refuses another file with the same `globe:`. Below, data/
+// means that folder. A checkout that still holds a globe in its own data/ is refused (nothing
+// started, nothing moved) with the command that moves it.
+//
+//   1. render: data/globe/ from <file> (or -f <file>), or the file that owns the snowglobe called
+//      <name> (snowglobe ls lists them), else SG_FILE, else the nearest snowglobe.yml here or above
+//      (none: up refuses; `up --default` runs the default globe, snowglobe.default.yml: one PDS,
+//      a PLC directory and Jetstream).
+//      SG_DOMAIN, from the environment or .env, replaces the file's domain.
+//      A group host with no image in the file is built from its build folder, else from
+//      SG_OPENSOCIAL_BUILD (a checkout of atproto-group-host); with neither, up stops here.
+//   2. certs, if data/https has no leaf yet, or it lacks a name (the user's CA, in ca/ beside the
+//      snowglobes' folders, is made once and kept)
+//   3. nginx's app routes and the network page, from data/https/apps (app --render), then each
+//      group host built from a folder is built again (Docker's cache keeps that quick)
+//   4. pass 1: up. The base init seeds alice, bob and SG_INVITE_CODE into
+//      data/accounts.env on the PDS with bare handles (else the first), as it does for the
+//      http stacks.
+//   5. the lexicon authority, lex-authority on the file's lexicons.authority PDS
+//      (LEX_AUTHORITY_* in data/accounts.env), then pass 2: up with
+//      SG_LEXICON_AUTHORITY_DID, which recreates the PDSes with it
+//   6. what the file seeds: each of its accounts (snowglobe account; the login under <NAME>_* in
+//      data/accounts.env, <NAME>_<PDS>_* on a PDS other than the one with bare handles, else the
+//      first), and its lexicons.publish folder into the authority (snowglobe lexicons). Its apps'
+//      local routes (port, handle ending) are set before step 2, so the leaf and nginx have them.
+//   7. an old data/devnet.env, from before snowglobe env, is removed; nothing writes one now
+//
+// A rerun finds the accounts and leaves unchanged lexicons alone. Anything else an app makes
+// (more accounts, invite codes) is its own, with account, invite and lexicons. Nothing secret is
+// printed. The tool ports (SG_PLC_PORT, SG_JETSTREAM_PORT, SG_JETSTREAM_METRICS_PORT,
+// SG_MAILDEV_WEB_PORT, SG_MAILDEV_SMTP_PORT, SG_PROXY_PORT) are published on the snowglobe's own
+// loopback address (SG_BIND, else the one chosen at its first up and kept in its state folder), so
+// two snowglobes need no port settings; a port variable moves one, in the environment or .env. devnet-spaces, the shared devnet, is refused (exit 3) as
+// the project unless SG_RESET_OK=1, before anything else is done.
+
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { appEnding, appEndingMessage, formatApps, readApps, renderRoutes, writeApps } from '../apps.mjs';
+import { loadGlobe } from '../config.mjs';
+import { ROOT, addName, chooseBind, dc, ensureSecrets, projectHasData, snowglobeRequest, envGet, guardProject, makePrivate, oldDataNotice, paths, renamedDefaultNotice, resolveBind, resolveDomain } from '../env.mjs';
+import { prebuilt } from '../prebuilt.mjs';
+import { byName, claimState, pointedAt, session } from '../state.mjs';
+import { accountPrefixes, endingClash, mergeRoutes } from '../up.mjs';
+import { account } from './account.mjs';
+import { lexicons } from './lexicons.mjs';
+import { checkedGlobe } from './check.mjs';
+import { writeGlobe } from './render.mjs';
+
+export const command = {
+  name: 'up',
+  usage: 'up [<file> | <name>] [-f <file>] [--default]',
+  // --default is the only way up runs the repo's default snowglobe; with no file and no
+  // snowglobe.yml in this folder or above, up refuses.
+  options: { default: { type: 'boolean' }, file: { type: 'string', short: 'f' } },
+  positionals: [0, 1],
+  accepts: ({ values, positionals }) => !(values.default && (positionals.length || values.file)),
+};
+
+// How long a one-shot (init, relay-init) may take, and nginx to answer again after a restart.
+const ONESHOT_SECONDS = 300;
+const NGINX_SECONDS = 30;
+
+// Until nginx answers the globe's page over TLS, with any status.
+async function waitServing(domain, { root, state, env }) {
+  const seconds = Number(env.SG_NGINX_WAIT) || NGINX_SECONDS;
+  for (let n = 0; ; n++) {
+    try {
+      return await snowglobeRequest(`https://${domain}/`, { root, state, env });
+    } catch (e) {
+      if (n >= seconds) throw new Error(`nginx did not answer https://${domain}/ within ${seconds} s after its restart: ${e.message}`);
+      await sleep(1000);
+    }
+  }
+}
+
+export async function run({ values, positionals: [word] }) {
+  let file;
+  if (values.default) file = join(ROOT, 'snowglobe.default.yml');
+  else {
+    const at = pointedAt({ file: values.file, word });
+    file = at.file;
+    if (at.name) {
+      // A snowglobe by its name is brought up from the file that owns it.
+      file = byName(at.name).path;
+      if (!file) throw new Error(`the file that owned the snowglobe ${at.name} is gone; bring it up from a file: snowglobe up -f <file>`);
+      // The file may have been renamed since (globe: changed): up would then start the new name.
+      const { globe } = loadGlobe(file);
+      if (globe && globe.globe !== at.name)
+        throw new Error(`${file} says globe: ${globe.globe} now, not ${at.name}; put globe: ${at.name} back to bring ${at.name} up, or snowglobe up ${file} to start ${globe.globe} as a new snowglobe`);
+    }
+  }
+  await up({ file, log: (m) => console.error(m) });
+}
+
+export async function up({ file, root = ROOT, env = process.env, log = () => {} }) {
+  // The shared devnet is refused before anything is looked up, even by a file that is not there.
+  if (env.SG_PROJECT) guardProject(env);
+  const found = session({ file, env, allowDefault: false });
+  const project = guardProject(env, found.name);
+  const state = found.state;
+  const old = oldDataNotice(root, state);
+  if (old) throw new Error(old);
+  const renamed = resolve(found.path) === join(root, 'snowglobe.default.yml') && renamedDefaultNotice(root, state);
+  if (renamed) throw new Error(renamed);
+  const p = paths(root, state);
+  // Both are read as compose reads them, and refused here when they are not a domain and a
+  // loopback address.
+  const { set } = resolveDomain({ root, state, env, rendering: true });
+  resolveBind({ root, state, env });
+
+  const { path, globe } = checkedGlobe(found.path, { domain: set || undefined });
+  if (!globe) throw new Error(`${path} has problems (above); ${p.globe} is as it was`);
+  // From here on the file's globe is the one this state folder is for.
+  const took = claimState(found);
+  if (took) log(took);
+  // Its own loopback address for every published port, chosen once, so a second snowglobe starts beside this one.
+  const bind = await chooseBind({ root, state, project, env });
+  if (bind !== '127.0.0.1') log(`${found.name} publishes its ports on ${bind}`);
+  // Checked before the render changes, so a refused file leaves the running globe's render.
+  const kept = readApps(p);
+  const badEnding = appEnding(kept);
+  if (badEnding) throw new Error(appEndingMessage(p.apps, badEnding));
+  const routes = mergeRoutes(kept, globe.apps);
+  const home = (globe.pds.find((x) => x.bare_handles) ?? globe.pds[0]).name;
+  const seedAccounts = accountPrefixes(globe.accounts, home);
+  const clash = endingClash(
+    [...globe.pds, ...globe.grouphosts].map((x) => x.name),
+    routes,
+  );
+  if (clash)
+    throw new Error(
+      `the server ${clash.pds} would take the handles under ${clash.pds}.<domain>, the ending app ${clash.app} claimed; rename the server, or snowglobe app --remove ${clash.app}`,
+    );
+  if (formatApps(routes) !== formatApps(kept)) {
+    mkdirSync(p.https, { recursive: true });
+    writeApps(p, routes);
+  }
+  writeGlobe(globe, path, p.globe, env, state);
+  const domain = globe.domain;
+  const ghs = globe.grouphosts.length ? `, group hosts ${globe.grouphosts.map((x) => x.name).join(' ')}` : '';
+  log(`${path}: globe ${globe.globe} at ${domain}, PDSs ${globe.pds.map((x) => x.name).join(' ')}${ghs}`);
+
+  // The leaf must carry every name (lib/certs.mjs): a leaf from before the app routes moved lacks
+  // *.app.<domain>, and one from before an app claimed a handle ending lacks its wildcard.
+  const { ensureCerts, leafLacks, leafNames } = await import('../certs.mjs');
+  const names = leafNames(p, domain, env.HTTPS_EXTRA_NAMES);
+  const newCerts = leafLacks(p.https, names, p.caDir);
+  if (newCerts) await ensureCerts({ dir: p.https, caDir: p.caDir, domain, names, log });
+  renderRoutes({ root, state, env });
+
+  const facts = (name) => envGet(name, p.globeEnv).split(' ').filter(Boolean);
+  // compose fills the file's ${SG_LEXICON_AUTHORITY_DID} from this.
+  const secrets = ensureSecrets({ root, state, env, hasData: () => projectHasData(project, env) });
+  if (secrets.kept.length)
+    log(`${found.name} has data made before its secrets were random, so it keeps the fixed ${secrets.kept.join(', ')}. snowglobe down -v, then up, starts it over with random ones (every account is lost).`);
+  const composeEnv = { ...env, ...secrets.values };
+  const compose = (args, opts = {}) => {
+    const res = dc(project, args, { root, state, env: composeEnv, ...opts });
+    if (res.error) throw res.error;
+    if (res.status !== 0)
+      throw new Error(`docker compose ${args.join(' ')} failed (exit ${res.status}). If a container crash-loops after you updated snowglobe, an image built earlier may be stale (the slingshot image is tagged by its commit, so a changed stack/slingshot/ folder alone does not rebuild it): run snowglobe down (without -v, which keeps your data), remove the image with docker rmi, then run up again`);
+    return res;
+  };
+  // The nginx template as the running nginx last started with it (stamped after a restart here).
+  const templateFile = join(root, 'stack/https/nginx.conf.template');
+  const templateStamp = join(p.https, 'nginx-template.sha256');
+  const templateHash = createHash('sha256').update(existsSync(templateFile) ? readFileSync(templateFile) : '').digest('hex');
+  const templateChanged = !existsSync(templateStamp) || readFileSync(templateStamp, 'utf8').trim() !== templateHash;
+  const put = (file, text) => {
+    writeFileSync(`${file}.tmp`, `${text}\n`);
+    renameSync(`${file}.tmp`, file);
+  };
+  // Container ids, none when compose cannot say (a project not made yet).
+  const ids = (args) => (dc(project, ['ps', ...args], { root, state, stdio: 'pipe', encoding: 'utf8' }).stdout ?? '').split('\n').filter(Boolean);
+
+  // `up --wait` alone fails as soon as a one-shot service exits, even with 0. So: up, then wait
+  // for the one-shots (init seeds data/accounts.env; relay-init registers every PDS with the
+  // relay), then for every long-running service to be healthy.
+  async function waitDone(service) {
+    for (let n = 0; n < ONESHOT_SECONDS; n++) {
+      const st = compose(['ps', '-a', '--format', '{{.State}} {{.ExitCode}}', service], { stdio: 'pipe', encoding: 'utf8' }).stdout.split('\n')[0];
+      if (st === 'exited 0') return;
+      if (st.startsWith('exited')) throw new Error(`${service} failed (exit ${st.slice('exited '.length)})`);
+      await sleep(1000);
+    }
+    throw new Error(`${service} did not finish within ${ONESHOT_SECONDS} s`);
+  }
+  const waitServices = () => compose(['up', '-d', '--wait', ...facts('SG_SERVICES')]);
+  async function upStack() {
+    // A service the file no longer runs (a PDS taken out, a service turned off) is stopped.
+    compose(['up', '-d', '--remove-orphans']);
+    for (const s of facts('SG_ONESHOTS')) await waitDone(s);
+    waitServices();
+  }
+
+  // What runs before this run: the relay, and every container, to notice a PDS this run makes.
+  const relayBefore = ids(['-q', 'relay']);
+  const before = new Set(ids(['-aq']));
+
+  // Every image the file builds is built before anything starts, so `up` never builds while
+  // containers are being made (on a runner with no images, the relay and the other built services
+  // were made, then made again by the next `up`, and Jetstream lost the relay between the two).
+  // A group host built from a folder is the folder as it is now; Docker's cache makes an
+  // unchanged image quick.
+  const builds = facts('SG_BUILDS');
+  // The images snowglobe publishes are pulled where the registry has the build of these very files
+  // (lib/prebuilt.mjs); only the rest are built. Compose fills in the ${...} defaults (an image's
+  // tag follows its commit); if it cannot say, everything is built, as before.
+  let toBuild = [];
+  try {
+    const services = JSON.parse(compose(['config', '--format', 'json'], { stdio: 'pipe', encoding: 'utf8' }).stdout).services;
+    const have = prebuilt({ root, services, env, log });
+    toBuild = Object.keys(services).filter((s) => services[s].build && !have.includes(s));
+  } catch {
+    toBuild = [];
+    compose(['build', '--quiet']);
+  }
+  if (toBuild.length) {
+    log(`Building images${builds.length ? ` (${builds.join(' ')})` : ''}`);
+    compose(['build', '--quiet', ...toBuild]);
+  }
+
+  // A rerun starts the PDSes with the authority they already have, so pass 1 changes nothing.
+  composeEnv.SG_LEXICON_AUTHORITY_DID = envGet('LEX_AUTHORITY_DID', p.accounts);
+  log(`Pass 1: up (project ${project})`);
+  await upStack();
+  // A running nginx keeps the certificate files it started with, and the template it was started
+  // with (the image fills it in at start); new ones need a restart, whether or not the leaf was
+  // reissued. The globe's servers may have changed with the render, so a running nginx reloads them.
+  if (newCerts || templateChanged) {
+    compose(['restart', 'nginx'], { stdio: 'ignore' });
+    if (templateChanged) put(templateStamp, templateHash);
+    // Its health check is nginx -t, which passes before it listens again, and the next step
+    // reaches the PDSes through it.
+    await waitServing(domain, { root, state, env });
+  } else compose(['exec', '-T', 'nginx', 'nginx', '-s', 'reload'], { stdio: 'ignore' });
+
+  const authority = envGet('SG_AUTHORITY_PDS', p.globeEnv);
+  const did = await account({ name: 'lex-authority', prefix: 'LEX_AUTHORITY', pds: authority, file: p.accounts, root, state, env, log });
+  if (did !== composeEnv.SG_LEXICON_AUTHORITY_DID) {
+    composeEnv.SG_LEXICON_AUTHORITY_DID = did;
+    log(`Pass 2: up with the lexicon authority ${did}`);
+    // Pass 2 recreates every PDS. Jetstream with no relay reads the first PDS itself and exits
+    // while it is gone; on a slow machine (a CI runner) it was still restarting when compose
+    // checked it, and up failed with "jetstream is unhealthy". Stopped first, it is started
+    // again only once the PDS is healthy.
+    const running = facts('SG_SERVICES');
+    if (running.includes('jetstream') && !running.includes('relay')) compose(['stop', 'jetstream'], { stdio: 'ignore' });
+    await upStack();
+  }
+
+  // A relay that was running when a PDS went away redials it with a growing backoff; a PDS
+  // added back under that name was not crawled again until the relay restarted (2026-10-20: no
+  // dial in the nine minutes after it came back). So when this run made a PDS container beside a
+  // relay that was already running, the relay restarts, and the services wait for it again.
+  if (relayBefore.length) {
+    const fresh = facts('SG_PDS').find((x) => ids(['-q', `pds-${x}`]).some((id) => !before.has(id)));
+    if (fresh) {
+      log(`Restarting the relay: pds-${fresh} is new beside it`);
+      compose(['restart', 'relay'], { stdio: 'ignore' });
+      waitServices();
+    }
+  }
+
+  // init's accounts are on the seed PDS; run maps their handles.
+  for (const prefix of ['ALICE', 'BOB']) {
+    const handle = envGet(`${prefix}_HANDLE`, p.accounts);
+    if (handle) addName(handle, { root, state });
+  }
+
+  // What the file seeds. A rerun finds each account by its kept login.
+  for (const a of seedAccounts) await account({ ...a, file: p.accounts, root, state, env, log });
+  if (globe.lexicons.publish) await lexicons({ dir: resolve(dirname(path), globe.lexicons.publish), root, state, env, log });
+
+  // Written by snowglobe up until snowglobe env replaced it; an app that still sources it would
+  // read addresses that no longer follow the file.
+  const denv = join(p.data, 'devnet.env');
+  if (existsSync(denv)) {
+    rmSync(denv);
+    log(`Removed the old ${denv}; snowglobe env hands an app these settings now`);
+  }
+
+  // init leaves data/accounts.env and data/accounts.json world-readable on every up; both hold
+  // logins.
+  makePrivate(p.accounts);
+  makePrivate(join(p.data, 'accounts.json'));
+
+  log('');
+  const ps = dc(project, ['ps', '--format', '{{.Service}} {{.State}} {{.Ports}}'], { root, state, stdio: 'pipe', encoding: 'utf8' });
+  log(ps.stdout.trimEnd());
+}