@openmeet/snowglobe 0.3.0 → 0.4.0

package/README.md

@@ -1,7 +1,7 @@
-# Snowglobe
+# <img src="assets/snowglobe.svg" width="40" alt=""> Snowglobe
 
 Your own isolated [AT Protocol](https://atproto.com) network, for testing, research, and demos. A snowglobe has its own
-PDS, PLC directory and Jetstream (and a relay, once there is more than one PDS), and nothing in it reaches Bluesky's network. It uses
+PDS, PLC directory, relay and Jetstream, and nothing in it reaches Bluesky's network. It uses
 https and the real network's sign-in (OAuth), so an app built for the real network runs against it
 with no test-only code.
 

package/lib/commands/up.mjs

@@ -41,6 +41,7 @@
 // two snowglobes need no port settings; a port variable moves one, in the environment or .env. devnet-spaces, the shared devnet, is refused (exit 3) as
 // the project unless SG_RESET_OK=1, before anything else is done.
 
+import { spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
@@ -50,7 +51,7 @@ import { loadGlobe } from '../config.mjs';
 import { ROOT, addName, chooseBind, dc, ensureSecrets, projectHasData, snowglobeRequest, envGet, guardProject, makePrivate, oldDataNotice, paths, renamedDefaultNotice, resolveBind, resolveDomain } from '../env.mjs';
 import { prebuilt } from '../prebuilt.mjs';
 import { byName, claimState, pointedAt, session } from '../state.mjs';
-import { accountPrefixes, endingClash, mergeRoutes } from '../up.mjs';
+import { accountPrefixes, endingClash, mergeRoutes, startedAfterRelay } from '../up.mjs';
 import { account } from './account.mjs';
 import { lexicons } from './lexicons.mjs';
 import { checkedGlobe } from './check.mjs';
@@ -203,10 +204,6 @@ export async function up({ file, root = ROOT, env = process.env, log = () => {}
     waitServices();
   }
 
-  // What runs before this run: the relay, and every container, to notice a PDS this run makes.
-  const relayBefore = ids(['-q', 'relay']);
-  const before = new Set(ids(['-aq']));
-
   // Every image the file builds is built before anything starts, so `up` never builds while
   // containers are being made (on a runner with no images, the relay and the other built services
   // were made, then made again by the next `up`, and Jetstream lost the relay between the two).
@@ -250,23 +247,27 @@ export async function up({ file, root = ROOT, env = process.env, log = () => {}
   if (did !== composeEnv.SG_LEXICON_AUTHORITY_DID) {
     composeEnv.SG_LEXICON_AUTHORITY_DID = did;
     log(`Pass 2: up with the lexicon authority ${did}`);
-    // Pass 2 recreates every PDS. Jetstream with no relay reads the first PDS itself and exits
-    // while it is gone; on a slow machine (a CI runner) it was still restarting when compose
-    // checked it, and up failed with "jetstream is unhealthy". Stopped first, it is started
-    // again only once the PDS is healthy.
-    const running = facts('SG_SERVICES');
-    if (running.includes('jetstream') && !running.includes('relay')) compose(['stop', 'jetstream'], { stdio: 'ignore' });
+    // Pass 2 recreates every PDS, so the relay restarts after it (below).
     await upStack();
   }
 
-  // A relay that was running when a PDS went away redials it with a growing backoff; a PDS
-  // added back under that name was not crawled again until the relay restarted (2026-10-20: no
-  // dial in the nine minutes after it came back). So when this run made a PDS container beside a
-  // relay that was already running, the relay restarts, and the services wait for it again.
-  if (relayBefore.length) {
-    const fresh = facts('SG_PDS').find((x) => ids(['-q', `pds-${x}`]).some((id) => !before.has(id)));
-    if (fresh) {
-      log(`Restarting the relay: pds-${fresh} is new beside it`);
+  // A relay that was running when a PDS, or nginx in front of it, went away redials it, and after
+  // ten quick failures it waits 30 s between dials: what the PDS writes meanwhile reaches it half a
+  // minute late, or not for minutes. A PDS added back was not dialed in the nine minutes after it
+  // came back (2026-10-20), and a record written right after a first up in CI had not reached Tap
+  // 20 s later (2026-10-30). Pass 2 recreates every PDS, new certificates restart nginx, and a rerun
+  // can make a PDS. So when one of them started after the relay did, the relay restarts, and the
+  // services wait for it again.
+  if (ids(['-q', 'relay']).length) {
+    const started = (service) => {
+      const containers = ids(['-q', service]);
+      if (!containers.length) return [];
+      return (spawnSync('docker', ['inspect', '-f', '{{.State.StartedAt}}', ...containers], { encoding: 'utf8', env }).stdout ?? '').split('\n').filter(Boolean);
+    };
+    const dialed = Object.fromEntries([...facts('SG_PDS').map((x) => `pds-${x}`), 'nginx'].map((s) => [s, started(s)]));
+    const later = startedAfterRelay(started('relay'), dialed);
+    if (later) {
+      log(`Restarting the relay: ${later} started after it`);
       compose(['restart', 'relay'], { stdio: 'ignore' });
       waitServices();
     }

package/lib/config.mjs

@@ -231,11 +231,13 @@ export function checkGlobe(doc, { baseDir } = {}) {
       }
     }
   }
-  // Without a relay Jetstream and TAP follow one PDS's own firehose, and the others' writes
-  // are seen by nothing.
-  if (services.relay === 'auto') services.relay = pds.length > 1;
+  // Without a relay TAP follows one PDS's own firehose, and the others' writes are seen by
+  // nothing. Jetstream needs the relay even with one PDS: at start it asks the relay for the
+  // network's hosts (com.atproto.sync.listHosts), which a PDS refuses.
+  if (services.relay === 'auto') services.relay = pds.length > 1 || services.jetstream;
   else if (services.relay === false && pds.length > 1)
     err('services.relay', 'cannot be off with more than one PDS: only one PDS would be followed');
+  else if (services.relay === false && services.jetstream) err('services.jetstream', 'needs the relay');
   // Slingshot reads this network's Jetstream.
   if (services.slingshot && !services.jetstream) err('services.slingshot', 'needs jetstream');
 

package/lib/render.mjs

@@ -46,6 +46,7 @@ const BUILT = {
   'snowglobe-dns': ['stack/https/dns', ['Dockerfile', 'dns.sh']],
   'snowglobe-proxy': ['stack/https/proxy', ['Dockerfile', 'server.mjs']],
   'snowglobe-pdsls': ['stack/pdsls', ['Dockerfile', 'network.patch', 'server.mjs']],
+  'snowglobe-jetstream': ['stack/jetstream', ['Dockerfile']],
 };
 
 /** The base compose files' services, which the globe's shared services start from. */
@@ -100,7 +101,8 @@ export function renderGlobe(globe, base, { stateDir, globeDir = join(stateDir ??
   // init seeds alice, bob and the invite code on the PDS with bare handles, else the first.
   const seed = globe.pds.find((p) => p.bare_handles) ?? globe.pds[0];
   const on = globe.services;
-  // Where Jetstream and TAP read: the relay, or with one PDS that PDS's own firehose.
+  // Where Jetstream and TAP read: the relay, or with one PDS and no relay, that PDS's own
+  // firehose (TAP only: Jetstream always has the relay, see config.mjs).
   const feed = on.relay ? 'http://relay:2470' : `http://${svc(globe.pds[0])}:${PDS_PORT}`;
   const copy = (o) => structuredClone(o);
 
@@ -259,13 +261,14 @@ export function renderGlobe(globe, base, { stateDir, globeDir = join(stateDir ??
   }
 
   if (on.jetstream) {
+    // Its first backfill asks each PDS for its repos at the PDS's https name, through nginx.
     services.jetstream = {
       ...copy(base.base.jetstream),
-      depends_on: on.relay ? { relay: healthy } : { [svc(globe.pds[0])]: healthy },
-      environment: {
-        ...copy(base.base.jetstream.environment),
-        JETSTREAM_WS_URL: `${feed.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`,
-      },
+      build: './stack/jetstream',
+      image: base.tags['snowglobe-jetstream'],
+      depends_on: { relay: healthy },
+      environment: { ...copy(base.base.jetstream.environment), JETSTREAM_RELAY_URL: feed },
+      volumes: [...base.base.jetstream.volumes, GO_CA],
       ports: [
         '${SG_BIND:-127.0.0.1}:${SG_JETSTREAM_PORT:-6008}:6008',
         '${SG_BIND:-127.0.0.1}:${SG_JETSTREAM_METRICS_PORT:-6009}:6009',

package/lib/up.mjs

@@ -1,6 +1,7 @@
 // What snowglobe up decides without Docker: the app routes the file adds, whether the globe's
 // PDSes clash with an app's handle ending, the env names the file's accounts keep their logins
-// under. Pure: inputs in, text or lists out (lib/commands/up.mjs runs the globe).
+// under, whether the relay restarts. Pure: inputs in, text or lists out (lib/commands/up.mjs runs
+// the globe).
 
 const envName = (label) => label.toUpperCase().replace(/-/g, '_');
 
@@ -52,3 +53,15 @@ export function endingClash(pdsNames, apps) {
   }
   return null;
 }
+
+/**
+ * The first of `dialed` (service name -> its containers' start times) with a container that started
+ * after the relay did (`relay`, its containers' start times), or null. The times are Docker's
+ * State.StartedAt. The relay dials each PDS through nginx, and one that was running when either
+ * went away can stop picking up what the PDS sends, so up restarts it.
+ */
+export function startedAfterRelay(relay, dialed) {
+  const since = Math.max(...relay.map((t) => Date.parse(t)));
+  for (const [service, times] of Object.entries(dialed)) if (times.some((t) => Date.parse(t) > since)) return service;
+  return null;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openmeet/snowglobe",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Standalone, testable ATProto development network for local development and CI",
   "type": "module",
   "bin": {
@@ -37,8 +37,8 @@
     "node": ">=20"
   },
   "dependencies": {
-    "@peculiar/asn1-schema": "2.11.0",
-    "@peculiar/asn1-x509": "2.11.0",
+    "@peculiar/asn1-schema": "^2.11.0",
+    "@peculiar/asn1-x509": "^2.11.0",
     "@peculiar/x509": "2.2.0",
     "reflect-metadata": "0.2.2",
     "yaml": "^2.9.1"

package/snowglobe.default.yml

@@ -14,8 +14,8 @@ pds:
   - name: main                 # https://main.local.snowglobe.at
     image: ghcr.io/bluesky-social/pds:0.4.5037    # a current release
     bare_handles: true         # its handles are <account>.local.snowglobe.at
-  # A second PDS, with handles <name>.other.local.snowglobe.at. With more than one PDS the
-  # relay turns on, so Jetstream sees every PDS's writes.
+  # A second PDS, with handles <name>.other.local.snowglobe.at. The relay crawls it too, so
+  # Jetstream sees its writes.
   # - name: other
   #   image: ghcr.io/bluesky-social/pds:0.4.5009
   #
@@ -30,7 +30,7 @@ services:                      # these are the defaults, each true or false
   slingshot: true              # answers a handle's DID and PDS inside the network; needs jetstream
   pdsls: true                  # https://pdsls.local.snowglobe.at, a browser for the network's records
   tap: false                   # sends an app the events of one collection (true, or a map; see snowglobe.md above)
-  # relay: auto                # auto: on when there is more than one PDS
+  # relay: auto                # auto: on with Jetstream, which reads it, or more than one PDS
 
 # Where every PDS resolves lexicons (NSIDs) from. Name a PDS that is in this file.
 # lexicons:

package/stack/docker-compose.yml

@@ -98,17 +98,21 @@ services:
       timeout: 3s
       retries: 10
 
-  # ── Jetstream (JSON event stream from PDS firehose) ──────────────
+  # ── Jetstream (JSON event stream from the relay's firehose) ──────
+  # Built from stack/jetstream: the pinned Jetstream, plus busybox for the health check. It reads
+  # a relay, not a PDS: at start it asks the relay for the network's hosts, which a PDS refuses.
   jetstream:
-    image: ghcr.io/bluesky-social/jetstream:sha-05c89f0
+    build: ./jetstream
+    image: snowglobe-jetstream
     restart: unless-stopped
-    depends_on:
-      pds:
-        condition: service_healthy
     environment:
-      JETSTREAM_WS_URL: ws://pds:3000/xrpc/com.atproto.sync.subscribeRepos
-      JETSTREAM_DATA_DIR: /data
-      JETSTREAM_LIVENESS_TTL: 86400s
+      JETSTREAM_RELAY_URL: http://relay:2470
+      JETSTREAM_PLC_URL: http://plc:2582
+      # A folder of its own: a volume from an older snowglobe holds the older Jetstream's files.
+      JETSTREAM_DATA_DIR: /data/v2
+      JETSTREAM_ADDR: ":6008"
+      # Metrics, at /metrics.
+      JETSTREAM_DEBUG_ADDR: ":6009"
     volumes:
       - jetstream-data:/data
     ports:
@@ -116,11 +120,15 @@ services:
       - "${SG_JETSTREAM_METRICS_PORT:-${DEVNET_JETSTREAM_METRICS_PORT:-6009}}:6009"
     networks:
       - atproto-devnet
+    # Healthy once it serves: its archive answers 503 until then, as its streams do. The debug
+    # port's /readyz says only that it is listening.
     healthcheck:
-      test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/127.0.0.1/6008'"]
+      test: ["CMD", "/busybox", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:6008/xrpc/network.bsky.jetstream.listSegments?limit=1"]
       interval: 5s
       timeout: 3s
       retries: 10
+      # A new snowglobe's first start takes a minute or two (docs/how-it-works.md).
+      start_period: 120s
 
   # ── TAP (repo sync + backfill) ───────────────────────────────────
   tap:

package/stack/jetstream/Dockerfile

@@ -0,0 +1,12 @@
+# Jetstream (github.com/bluesky-social/jetstream), the firehose as JSON for apps. It serves v2,
+# /xrpc/network.bsky.jetstream.subscribeEvents, which contrail and @bsky/jetstream read, and the
+# older /subscribe, which Slingshot, PDSls and the network page read.
+# To move to another Jetstream, change the second FROM line. snowglobe up builds the new image,
+# since the image's tag is a hash of this file.
+# The image has no shell, so busybox comes along for the health check. It runs as root so it can
+# write the jetstream-data volume, which Docker makes root's.
+FROM busybox:1.37.0-musl@sha256:f6a95d611e0a8780fabdb2aec43e31275b2720dfefa3e41b16835bfc42454478 AS tools
+
+FROM ghcr.io/bluesky-social/jetstream:v0.4.6@sha256:38e927566ed2aca19ffd2e9f3c440db74ffb404000aaf493b215dfefc6c7e54c
+COPY --from=tools /bin/busybox /busybox
+USER 0:0