@openmeet/snowglobe 0.3.0-rc.1 → 0.4.0

package/README.md

@@ -1,7 +1,7 @@
-# Snowglobe
+# <img src="assets/snowglobe.svg" width="40" alt=""> Snowglobe
 
 Your own isolated [AT Protocol](https://atproto.com) network, for testing, research, and demos. A snowglobe has its own
-PDS, PLC directory and Jetstream (and a relay, once there is more than one PDS), and nothing in it reaches Bluesky's network. It uses
+PDS, PLC directory, relay and Jetstream, and nothing in it reaches Bluesky's network. It uses
 https and the real network's sign-in (OAuth), so an app built for the real network runs against it
 with no test-only code.
 

package/lib/apps.mjs

@@ -15,7 +15,7 @@ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from '
 import { join } from 'node:path';
 import { ROOT, globeFacts, paths } from './env.mjs';
 
-const HEADER = '# Written by bin/snowglobe.mjs app from https/apps; rerun it instead of editing.\n';
+const HEADER = '# Written by snowglobe app from https/apps; rerun it instead of editing.\n';
 
 /**
  * The first saved route whose ending is `app`, or null. app.<domain> is where the apps are, so an
@@ -25,7 +25,7 @@ export const appEnding = (apps) => apps.find((a) => a.ending === 'app') ?? null;
 
 export const appEndingMessage = (file, a) =>
   `${file}: the line "${formatApps([a]).trim()}" claims the ending app, which is where the apps are (${a.name}.app.<domain>); ` +
-  `remove the route (bin/snowglobe.mjs app --remove ${a.name}) or give it another ending`;
+  `remove the route (snowglobe app --remove ${a.name}) or give it another ending`;
 
 /** The routes in an apps file's text: [{ name, port, ending }], ending null when none. */
 export function parseApps(text) {
@@ -85,7 +85,7 @@ export const writeApps = (p, apps) => put(p.apps, formatApps(apps));
 export function renderRoutes({ root = ROOT, state, env = process.env } = {}) {
   const p = paths(root, state);
   const pageJson = join(p.globe, 'page.json');
-  if (!existsSync(pageJson)) throw new Error(`No ${pageJson}: run bin/snowglobe.mjs up, which renders it`);
+  if (!existsSync(pageJson)) throw new Error(`No ${pageJson}: run snowglobe up, which renders it`);
   const { domain } = globeFacts({ root, state, env });
   const apps = readApps(p);
   const bad = appEnding(apps);

package/lib/commands/up.mjs

@@ -41,6 +41,7 @@
 // two snowglobes need no port settings; a port variable moves one, in the environment or .env. devnet-spaces, the shared devnet, is refused (exit 3) as
 // the project unless SG_RESET_OK=1, before anything else is done.
 
+import { spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
@@ -50,7 +51,7 @@ import { loadGlobe } from '../config.mjs';
 import { ROOT, addName, chooseBind, dc, ensureSecrets, projectHasData, snowglobeRequest, envGet, guardProject, makePrivate, oldDataNotice, paths, renamedDefaultNotice, resolveBind, resolveDomain } from '../env.mjs';
 import { prebuilt } from '../prebuilt.mjs';
 import { byName, claimState, pointedAt, session } from '../state.mjs';
-import { accountPrefixes, endingClash, mergeRoutes } from '../up.mjs';
+import { accountPrefixes, endingClash, mergeRoutes, startedAfterRelay } from '../up.mjs';
 import { account } from './account.mjs';
 import { lexicons } from './lexicons.mjs';
 import { checkedGlobe } from './check.mjs';
@@ -203,10 +204,6 @@ export async function up({ file, root = ROOT, env = process.env, log = () => {}
     waitServices();
   }
 
-  // What runs before this run: the relay, and every container, to notice a PDS this run makes.
-  const relayBefore = ids(['-q', 'relay']);
-  const before = new Set(ids(['-aq']));
-
   // Every image the file builds is built before anything starts, so `up` never builds while
   // containers are being made (on a runner with no images, the relay and the other built services
   // were made, then made again by the next `up`, and Jetstream lost the relay between the two).
@@ -250,23 +247,27 @@ export async function up({ file, root = ROOT, env = process.env, log = () => {}
   if (did !== composeEnv.SG_LEXICON_AUTHORITY_DID) {
     composeEnv.SG_LEXICON_AUTHORITY_DID = did;
     log(`Pass 2: up with the lexicon authority ${did}`);
-    // Pass 2 recreates every PDS. Jetstream with no relay reads the first PDS itself and exits
-    // while it is gone; on a slow machine (a CI runner) it was still restarting when compose
-    // checked it, and up failed with "jetstream is unhealthy". Stopped first, it is started
-    // again only once the PDS is healthy.
-    const running = facts('SG_SERVICES');
-    if (running.includes('jetstream') && !running.includes('relay')) compose(['stop', 'jetstream'], { stdio: 'ignore' });
+    // Pass 2 recreates every PDS, so the relay restarts after it (below).
     await upStack();
   }
 
-  // A relay that was running when a PDS went away redials it with a growing backoff; a PDS
-  // added back under that name was not crawled again until the relay restarted (2026-10-20: no
-  // dial in the nine minutes after it came back). So when this run made a PDS container beside a
-  // relay that was already running, the relay restarts, and the services wait for it again.
-  if (relayBefore.length) {
-    const fresh = facts('SG_PDS').find((x) => ids(['-q', `pds-${x}`]).some((id) => !before.has(id)));
-    if (fresh) {
-      log(`Restarting the relay: pds-${fresh} is new beside it`);
+  // A relay that was running when a PDS, or nginx in front of it, went away redials it, and after
+  // ten quick failures it waits 30 s between dials: what the PDS writes meanwhile reaches it half a
+  // minute late, or not for minutes. A PDS added back was not dialed in the nine minutes after it
+  // came back (2026-10-20), and a record written right after a first up in CI had not reached Tap
+  // 20 s later (2026-10-30). Pass 2 recreates every PDS, new certificates restart nginx, and a rerun
+  // can make a PDS. So when one of them started after the relay did, the relay restarts, and the
+  // services wait for it again.
+  if (ids(['-q', 'relay']).length) {
+    const started = (service) => {
+      const containers = ids(['-q', service]);
+      if (!containers.length) return [];
+      return (spawnSync('docker', ['inspect', '-f', '{{.State.StartedAt}}', ...containers], { encoding: 'utf8', env }).stdout ?? '').split('\n').filter(Boolean);
+    };
+    const dialed = Object.fromEntries([...facts('SG_PDS').map((x) => `pds-${x}`), 'nginx'].map((s) => [s, started(s)]));
+    const later = startedAfterRelay(started('relay'), dialed);
+    if (later) {
+      log(`Restarting the relay: ${later} started after it`);
       compose(['restart', 'relay'], { stdio: 'ignore' });
       waitServices();
     }

package/lib/config.mjs

@@ -231,11 +231,13 @@ export function checkGlobe(doc, { baseDir } = {}) {
       }
     }
   }
-  // Without a relay Jetstream and TAP follow one PDS's own firehose, and the others' writes
-  // are seen by nothing.
-  if (services.relay === 'auto') services.relay = pds.length > 1;
+  // Without a relay TAP follows one PDS's own firehose, and the others' writes are seen by
+  // nothing. Jetstream needs the relay even with one PDS: at start it asks the relay for the
+  // network's hosts (com.atproto.sync.listHosts), which a PDS refuses.
+  if (services.relay === 'auto') services.relay = pds.length > 1 || services.jetstream;
   else if (services.relay === false && pds.length > 1)
     err('services.relay', 'cannot be off with more than one PDS: only one PDS would be followed');
+  else if (services.relay === false && services.jetstream) err('services.jetstream', 'needs the relay');
   // Slingshot reads this network's Jetstream.
   if (services.slingshot && !services.jetstream) err('services.slingshot', 'needs jetstream');
 

package/lib/render.mjs

@@ -46,6 +46,7 @@ const BUILT = {
   'snowglobe-dns': ['stack/https/dns', ['Dockerfile', 'dns.sh']],
   'snowglobe-proxy': ['stack/https/proxy', ['Dockerfile', 'server.mjs']],
   'snowglobe-pdsls': ['stack/pdsls', ['Dockerfile', 'network.patch', 'server.mjs']],
+  'snowglobe-jetstream': ['stack/jetstream', ['Dockerfile']],
 };
 
 /** The base compose files' services, which the globe's shared services start from. */
@@ -100,7 +101,8 @@ export function renderGlobe(globe, base, { stateDir, globeDir = join(stateDir ??
   // init seeds alice, bob and the invite code on the PDS with bare handles, else the first.
   const seed = globe.pds.find((p) => p.bare_handles) ?? globe.pds[0];
   const on = globe.services;
-  // Where Jetstream and TAP read: the relay, or with one PDS that PDS's own firehose.
+  // Where Jetstream and TAP read: the relay, or with one PDS and no relay, that PDS's own
+  // firehose (TAP only: Jetstream always has the relay, see config.mjs).
   const feed = on.relay ? 'http://relay:2470' : `http://${svc(globe.pds[0])}:${PDS_PORT}`;
   const copy = (o) => structuredClone(o);
 
@@ -259,13 +261,14 @@ export function renderGlobe(globe, base, { stateDir, globeDir = join(stateDir ??
   }
 
   if (on.jetstream) {
+    // Its first backfill asks each PDS for its repos at the PDS's https name, through nginx.
     services.jetstream = {
       ...copy(base.base.jetstream),
-      depends_on: on.relay ? { relay: healthy } : { [svc(globe.pds[0])]: healthy },
-      environment: {
-        ...copy(base.base.jetstream.environment),
-        JETSTREAM_WS_URL: `${feed.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`,
-      },
+      build: './stack/jetstream',
+      image: base.tags['snowglobe-jetstream'],
+      depends_on: { relay: healthy },
+      environment: { ...copy(base.base.jetstream.environment), JETSTREAM_RELAY_URL: feed },
+      volumes: [...base.base.jetstream.volumes, GO_CA],
       ports: [
         '${SG_BIND:-127.0.0.1}:${SG_JETSTREAM_PORT:-6008}:6008',
         '${SG_BIND:-127.0.0.1}:${SG_JETSTREAM_METRICS_PORT:-6009}:6009',
@@ -452,14 +455,14 @@ export function renderGlobe(globe, base, { stateDir, globeDir = join(stateDir ??
   }
 
   const header = (c) =>
-    `${c} Written by bin/snowglobe.mjs render from ${source}; render again instead of editing.\n`;
+    `${c} Written by snowglobe render from ${source}; render again instead of editing.\n`;
 
   const compose =
     header('#') +
     `# The https globe "${globe.globe}" at ${D}: ${globe.pds.map((p) => p.name).join(', ')}` +
     (globe.grouphosts.length ? `; group hosts ${globe.grouphosts.map((g) => g.name).join(', ')}` : '') +
     '.\n' +
-    '# Run it with bin/snowglobe.mjs up, which renders this file first.\n\n' +
+    '# Run it with snowglobe up, which renders this file first.\n\n' +
     // YAML 1.1 quoting, so a value such as restart's "no" stays a string to compose.
     stringify({ services, volumes, networks: { [NETWORK]: null } }, { lineWidth: 0, nullStr: '', aliasDuplicateObjects: false, version: '1.1' });
 
@@ -514,15 +517,18 @@ export function renderGlobe(globe, base, { stateDir, globeDir = join(stateDir ??
     const named = Object.entries(IMAGE_NAMES).find(([, ref]) => ref === image)?.[0];
     return named ?? image.split('/').pop().replace(/@sha256:([0-9a-f]{12})[0-9a-f]+$/, '@sha256:$1…');
   };
+  // What the globe is made of, as the page's first line names it: only what runs.
+  const parts = [globe.pds.length === 1 ? 'PDS' : 'PDSs', 'PLC directory', on.relay && 'relay', on.jetstream && 'Jetstream'].filter(Boolean);
   const page = {
     globe: globe.globe,
     domain: D,
     about:
-      'An atproto network on this machine: its own PDSs, PLC directory, relay and Jetstream. Its accounts live in its own directory, not the public one, so Bluesky cannot find anything here. An app is handed this network\'s addresses (snowglobe env), not Bluesky\'s.',
+      `An atproto network on this machine: its own ${parts.slice(0, -1).join(', ')} and ${parts.at(-1)}. Its accounts live in its own directory, not the public one, so Bluesky cannot find anything here. An app is handed this network's addresses (snowglobe env), not Bluesky's.`,
     accountsNote:
       `Every account on the network, on each of its ${globe.pds.length} PDS${globe.pds.length === 1 ? '' : 's'}` +
       (globe.grouphosts.length ? `, and every group on its group host${globe.grouphosts.length === 1 ? '' : 's'}.` : '.'),
-    ...(on.jetstream ? { feed: '/subscribe' } : {}),
+    // Without Jetstream the page has no feed to follow, and says so instead of reconnecting.
+    feed: on.jetstream ? '/subscribe' : false,
     services: [
       ...globe.pds.map((p) => [
         'PDS',

package/lib/up.mjs

@@ -1,6 +1,7 @@
 // What snowglobe up decides without Docker: the app routes the file adds, whether the globe's
 // PDSes clash with an app's handle ending, the env names the file's accounts keep their logins
-// under. Pure: inputs in, text or lists out (lib/commands/up.mjs runs the globe).
+// under, whether the relay restarts. Pure: inputs in, text or lists out (lib/commands/up.mjs runs
+// the globe).
 
 const envName = (label) => label.toUpperCase().replace(/-/g, '_');
 
@@ -52,3 +53,15 @@ export function endingClash(pdsNames, apps) {
   }
   return null;
 }
+
+/**
+ * The first of `dialed` (service name -> its containers' start times) with a container that started
+ * after the relay did (`relay`, its containers' start times), or null. The times are Docker's
+ * State.StartedAt. The relay dials each PDS through nginx, and one that was running when either
+ * went away can stop picking up what the PDS sends, so up restarts it.
+ */
+export function startedAfterRelay(relay, dialed) {
+  const since = Math.max(...relay.map((t) => Date.parse(t)));
+  for (const [service, times] of Object.entries(dialed)) if (times.some((t) => Date.parse(t) > since)) return service;
+  return null;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openmeet/snowglobe",
-  "version": "0.3.0-rc.1",
+  "version": "0.4.0",
   "description": "Standalone, testable ATProto development network for local development and CI",
   "type": "module",
   "bin": {
@@ -37,8 +37,8 @@
     "node": ">=20"
   },
   "dependencies": {
-    "@peculiar/asn1-schema": "2.11.0",
-    "@peculiar/asn1-x509": "2.11.0",
+    "@peculiar/asn1-schema": "^2.11.0",
+    "@peculiar/asn1-x509": "^2.11.0",
     "@peculiar/x509": "2.2.0",
     "reflect-metadata": "0.2.2",
     "yaml": "^2.9.1"

package/snowglobe.default.yml

@@ -1,8 +1,11 @@
 # The default snowglobe: one current PDS, a PLC directory and Jetstream, all at https names, with
 # Slingshot and PDSls (the record browser) beside them.
-# Copy this file to snowglobe.yml (here, or in your app's repo) and run bin/snowglobe.mjs up.
-# The commented sections below add more; docs/snowglobe.md says what each one does, and
-# docs/examples/atmo.yml is the whole thing: three PDSes, a relay and a lexicon authority.
+# Copy this file to snowglobe.yml (here, or in your app's repo) and run snowglobe up
+# (bin/snowglobe.mjs up from a checkout of snowglobe). The commented sections below add more.
+# What each one does:
+#   https://github.com/OpenMeet-Team/snowglobe/blob/main/docs/snowglobe.md
+# The whole thing, three PDSes, a relay and a lexicon authority:
+#   https://github.com/OpenMeet-Team/snowglobe/blob/main/docs/examples/atmo.yml
 
 globe: snowglobe
 domain: local.snowglobe.at     # every name below is under it
@@ -11,8 +14,8 @@ pds:
   - name: main                 # https://main.local.snowglobe.at
     image: ghcr.io/bluesky-social/pds:0.4.5037    # a current release
     bare_handles: true         # its handles are <account>.local.snowglobe.at
-  # A second PDS, with handles <name>.other.local.snowglobe.at. With more than one PDS the
-  # relay turns on, so Jetstream sees every PDS's writes.
+  # A second PDS, with handles <name>.other.local.snowglobe.at. The relay crawls it too, so
+  # Jetstream sees its writes.
   # - name: other
   #   image: ghcr.io/bluesky-social/pds:0.4.5009
   #
@@ -26,8 +29,8 @@ services:                      # these are the defaults, each true or false
   jetstream: true
   slingshot: true              # answers a handle's DID and PDS inside the network; needs jetstream
   pdsls: true                  # https://pdsls.local.snowglobe.at, a browser for the network's records
-  tap: false                   # sends an app the events of one collection (true, or a map; see docs/snowglobe.md)
-  # relay: auto                # auto: on when there is more than one PDS
+  tap: false                   # sends an app the events of one collection (true, or a map; see snowglobe.md above)
+  # relay: auto                # auto: on with Jetstream, which reads it, or more than one PDS
 
 # Where every PDS resolves lexicons (NSIDs) from. Name a PDS that is in this file.
 # lexicons:
@@ -45,7 +48,8 @@ accounts:                      # <name>@<pds>; up makes each one
   - bob@main
 
 # Your app, reached at https://myapp.app.local.snowglobe.at, with settings from the snowglobe.
-# `snowglobe env` hands them over; docs/your-app.md has the names.
+# `snowglobe env` hands them over. The names:
+#   https://github.com/OpenMeet-Team/snowglobe/blob/main/docs/your-app.md
 # apps:
 #   - name: myapp
 #     local: { port: 5173 }

package/stack/docker-compose.yml

@@ -98,17 +98,21 @@ services:
       timeout: 3s
       retries: 10
 
-  # ── Jetstream (JSON event stream from PDS firehose) ──────────────
+  # ── Jetstream (JSON event stream from the relay's firehose) ──────
+  # Built from stack/jetstream: the pinned Jetstream, plus busybox for the health check. It reads
+  # a relay, not a PDS: at start it asks the relay for the network's hosts, which a PDS refuses.
   jetstream:
-    image: ghcr.io/bluesky-social/jetstream:sha-05c89f0
+    build: ./jetstream
+    image: snowglobe-jetstream
     restart: unless-stopped
-    depends_on:
-      pds:
-        condition: service_healthy
     environment:
-      JETSTREAM_WS_URL: ws://pds:3000/xrpc/com.atproto.sync.subscribeRepos
-      JETSTREAM_DATA_DIR: /data
-      JETSTREAM_LIVENESS_TTL: 86400s
+      JETSTREAM_RELAY_URL: http://relay:2470
+      JETSTREAM_PLC_URL: http://plc:2582
+      # A folder of its own: a volume from an older snowglobe holds the older Jetstream's files.
+      JETSTREAM_DATA_DIR: /data/v2
+      JETSTREAM_ADDR: ":6008"
+      # Metrics, at /metrics.
+      JETSTREAM_DEBUG_ADDR: ":6009"
     volumes:
       - jetstream-data:/data
     ports:
@@ -116,11 +120,15 @@ services:
       - "${SG_JETSTREAM_METRICS_PORT:-${DEVNET_JETSTREAM_METRICS_PORT:-6009}}:6009"
     networks:
       - atproto-devnet
+    # Healthy once it serves: its archive answers 503 until then, as its streams do. The debug
+    # port's /readyz says only that it is listening.
     healthcheck:
-      test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/127.0.0.1/6008'"]
+      test: ["CMD", "/busybox", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:6008/xrpc/network.bsky.jetstream.listSegments?limit=1"]
       interval: 5s
       timeout: 3s
       retries: 10
+      # A new snowglobe's first start takes a minute or two (docs/how-it-works.md).
+      start_period: 120s
 
   # ── TAP (repo sync + backfill) ───────────────────────────────────
   tap:

package/stack/https/nginx.conf.template

@@ -96,7 +96,7 @@ server {
     listen 443 ssl;
     server_name *.app.${SG_DOMAIN};
     if ($snowglobe_app = "") {
-        return 404 "No app is registered at $host (bin/snowglobe.mjs app).\n";
+        return 404 "No app is registered at $host (snowglobe app).\n";
     }
     location / {
         proxy_pass http://$snowglobe_app;

package/stack/jetstream/Dockerfile

@@ -0,0 +1,12 @@
+# Jetstream (github.com/bluesky-social/jetstream), the firehose as JSON for apps. It serves v2,
+# /xrpc/network.bsky.jetstream.subscribeEvents, which contrail and @bsky/jetstream read, and the
+# older /subscribe, which Slingshot, PDSls and the network page read.
+# To move to another Jetstream, change the second FROM line. snowglobe up builds the new image,
+# since the image's tag is a hash of this file.
+# The image has no shell, so busybox comes along for the health check. It runs as root so it can
+# write the jetstream-data volume, which Docker makes root's.
+FROM busybox:1.37.0-musl@sha256:f6a95d611e0a8780fabdb2aec43e31275b2720dfefa3e41b16835bfc42454478 AS tools
+
+FROM ghcr.io/bluesky-social/jetstream:v0.4.6@sha256:38e927566ed2aca19ffd2e9f3c440db74ffb404000aaf493b215dfefc6c7e54c
+COPY --from=tools /bin/busybox /busybox
+USER 0:0

package/stack/page/index.html

@@ -233,7 +233,8 @@
     if (tally.people != null) parts.push(plural(tally.people, "person", "people"));
     if (tally.groups) parts.push(plural(tally.groups, "group", "groups"));
     var line = parts.length ? parts.join(" and ") + ". " : "";
-    document.getElementById("tally").textContent = line + plural(tally.writes, "write", "writes") + " in the last 24 hours.";
+    // writes is null on a network with no feed to count them from.
+    document.getElementById("tally").textContent = line + (tally.writes == null ? "" : plural(tally.writes, "write", "writes") + " in the last 24 hours.");
   }
 
   function health(domain, services) {
@@ -380,6 +381,15 @@
     connect();
   }
 
+  // A network with no Jetstream has no feed: the panel says so instead of reconnecting forever.
+  function noActivity() {
+    document.getElementById("stream").style.display = "none";
+    document.getElementById("count").parentNode.style.display = "none";
+    var body = document.getElementById("events");
+    body.textContent = "";
+    add(body, el("p", { "class": "empty" }, "This network runs no Jetstream, so its writes are not followed here."));
+  }
+
   getJSON("/snowglobe.json").then(function (g) {
     document.title = g.globe + " · Snowglobe network";
     document.getElementById("title").textContent = g.globe;
@@ -396,12 +406,14 @@
     }
     if (g.about) document.getElementById("about").textContent = g.about;
     if (g.accountsNote) document.getElementById("accounts-note").textContent = g.accountsNote;
-    var feed = g.feed || "wss://jetstream." + g.domain + "/subscribe";
-    if (feed.charAt(0) === "/") feed = "wss://" + location.host + feed;
+    // feed false: the network runs no Jetstream. Unset, Jetstream is at its usual public name.
+    var feed = g.feed === false ? null : g.feed || "wss://jetstream." + g.domain + "/subscribe";
+    if (feed && feed.charAt(0) === "/") feed = "wss://" + location.host + feed;
+    if (!feed) { tally.writes = null; noActivity(); }
     showTally();
     health(g.domain, g.services || SERVICES);
     // The feed names accounts by handle, so it waits for the account list.
-    accounts(g.accounts || ACCOUNTS).catch(function () {}).then(function () { activity(feed); });
+    accounts(g.accounts || ACCOUNTS).catch(function () {}).then(function () { if (feed) activity(feed); });
   });
 })();
 </script>