heyquilt 0.3.90 → 0.3.107

package/src/api/relay-cluster.js

@@ -1,9 +1,12 @@
 // The relay on many machines, from the accounts API's side (docs/superpowers/specs/2026-10-17-relay-scaling.md).
 // Each room (session) lives on one relay machine. Machines say hello when they start and report
-// every minute (src/relay-cluster.js); a room nobody placed yet goes to the least loaded machine;
-// a heavy room gets a machine of its own, which the API makes through Fly's Machines API
-// (relay-machines.js) and then moves the room to. The API reaches one machine through the relay's
-// public address with `fly-force-instance-id`, so every call lands where it is meant to.
+// every minute (src/relay-cluster.js) with their own load (memory, event-loop lag, CPU, traffic)
+// and each room's. A room nobody placed yet goes to the least loaded machine under its load target;
+// a heavy room gets a machine of its own. The cluster scales both ways on that load: a machine
+// over its target for a while hands its lightest rooms to one under it (or a new one), machines
+// that could share one consolidate, and a shared machine left empty is stopped and destroyed with
+// its volume (relay-machines.js, Fly's Machines API), always keeping a minimum. The API reaches one
+// machine through the relay's public address with `fly-force-instance-id`.
 import { busyMinute, heavyReasons, HEAVY } from '../relay-metrics.js'
 
 // A machine that has not reported for this long takes no new rooms.
@@ -14,39 +17,150 @@ export const JOB_STALE_MS = 15 * 60 * 1000
 export const JOB_RETRY_MS = 10 * 60 * 1000
 // How long a new machine has to say hello before the job gives up on it.
 export const HELLO_WAIT_MS = 5 * 60 * 1000
+// A machine's rooms move one at a time; its next move waits for a whole minute's report after the last.
+export const MOVE_RETRY_MS = 90 * 1000
+// Between marking an empty machine retiring and destroying it: a placement or move that read it as
+// active a moment ago lands (and cancels the retire) or sees it retiring (and goes elsewhere) first.
+export const RETIRE_GRACE_MS = 60 * 1000
+// How often one API machine looks over the whole cluster (on a machine's report).
+export const TICK_MS = 30 * 1000
+const MIN = 60 * 1000
 const ROOM = /^[A-Za-z0-9_-]{1,64}$/
 const MACHINE = /^[A-Za-z0-9_-]{1,64}$/
 
-const alive = (m, t) => m.state === 'active' && Math.max(m.reportedAt || 0, m.helloAt || 0) >= t - ALIVE_MS
+/**
+ * The load a shared machine should stay under (settable on the API, relayTargetsFromEnv).
+ * Rooms placed (QUILT_RELAY_CAPACITY on the relay) are only a hard ceiling for new rooms.
+ */
+export const TARGETS = {
+  memory: 0.7, // memory used / the machine's memory
+  lagMs: 100, // event-loop lag (the minute's 99th percentile)
+  cpu: 0.9, // share of one core over the minute
+  hotMinutes: 3, // lag or CPU over target must last this many minutes running
+  bytesPerMin: 100 * 1024 * 1024, // WebSocket bytes in and out a minute, every room together
+  rebalanceMinutes: 5, // over target this many minutes running: its lightest rooms move off
+  emptyMinutes: 30, // a shared machine empty this long is stopped and destroyed
+  minMachines: 1, // shared machines always kept
+  consolidateBelow: 0.25, // two machines both under this share of their target: one hands its rooms to the other
+  moveGapMinutes: 30 // a room never moves again sooner than this
+}
+
+const ENV = {
+  memory: 'QUILT_RELAY_TARGET_MEMORY',
+  lagMs: 'QUILT_RELAY_TARGET_LAG_MS',
+  cpu: 'QUILT_RELAY_TARGET_CPU',
+  hotMinutes: 'QUILT_RELAY_TARGET_HOT_MINUTES',
+  bytesPerMin: 'QUILT_RELAY_TARGET_BYTES_PER_MIN',
+  rebalanceMinutes: 'QUILT_RELAY_REBALANCE_MINUTES',
+  emptyMinutes: 'QUILT_RELAY_EMPTY_MINUTES',
+  minMachines: 'QUILT_RELAY_MIN_MACHINES',
+  consolidateBelow: 'QUILT_RELAY_CONSOLIDATE_BELOW',
+  moveGapMinutes: 'QUILT_RELAY_MOVE_GAP_MINUTES'
+}
+
+/** TARGETS, with any of ENV's variables set to a number (fractions above 0, counts 0 or more) in their place. */
+export function relayTargetsFromEnv (env = process.env) {
+  const out = { ...TARGETS }
+  for (const [k, name] of Object.entries(ENV)) {
+    const raw = String(env[name] ?? '').trim()
+    if (!raw) continue
+    const v = Number(raw)
+    if (!Number.isFinite(v) || v < 0 || (v === 0 && !['minMachines', 'moveGapMinutes'].includes(k))) continue
+    out[k] = ['minMachines', 'hotMinutes', 'rebalanceMinutes'].includes(k) ? Math.round(v) : v
+  }
+  return out
+}
+
+/** Takes rooms (and is kept up): reported lately, and active or retiring (a retiring machine still serves what it holds). */
+const up = (m, t) => (m.state === 'active' || m.state === 'retiring') && Math.max(m.reportedAt || 0, m.helloAt || 0) >= t - ALIVE_MS
+const alive = (m, t) => m.state === 'active' && up(m, t)
+const num = (v) => (Number.isFinite(Number(v)) ? Number(v) : 0)
+
+/**
+ * Whether something that started at `since` (a report's time: that report covered the minute before
+ * it) has lasted `minutes` by `t`. Time, not a count of reports, so a relay reporting more often
+ * than once a minute changes nothing.
+ */
+export const lasted = (since, minutes, t) => since != null && t - since + MIN >= minutes * MIN
+
+/** Whether a machine's minute had its event loop or CPU over target (it must last `hotMinutes`). */
+export function hotMinute (load = {}, targets = TARGETS) {
+  return num(load.lagMs) >= targets.lagMs || (targets.cpu > 0 && num(load.cpu) >= targets.cpu)
+}
+
+/** Each part of a machine's load as a share of its target (1 is at target); `rooms` is rooms placed against its capacity. */
+export function loadShares (m, placed = 0, targets = TARGETS) {
+  const l = m.load || {}
+  const total = num(l.memTotal)
+  return {
+    memory: total > 0 ? num(l.memUsed) / total / targets.memory : 0,
+    lag: num(l.lagMs) / targets.lagMs,
+    cpu: targets.cpu > 0 ? num(l.cpu) / targets.cpu : 0,
+    io: num(l.bytesPerMin) / targets.bytesPerMin,
+    rooms: placed / (m.capacity || 1)
+  }
+}
+
+/** How loaded a machine is: its most loaded part, as a share of the target. */
+export const loadScore = (m, placed = 0, targets = TARGETS) => Math.max(...Object.values(loadShares(m, placed, targets)))
 
 /**
- * Where a new room goes: the live shared machine (not dedicated to a room) with the fewest rooms
- * loaded among those under their capacity, else the one with the fewest rooms placed. `full`
- * says every live shared machine is at capacity (time for another). Null when none is live.
+ * Why a machine is over its load target ('memory', 'io', 'lag'), or [] when it isn't. Lag and CPU
+ * count once they have lasted `hotMinutes` (since `hotAt`, the report that first had them over) by `t`.
  */
-export function chooseMachine (machines, counts, t) {
-  const live = machines.filter((m) => !m.dedicatedRoom && alive(m, t))
+export function overReasons (m, targets = TARGETS, t = Date.now()) {
+  const s = loadShares(m, 0, targets)
+  const out = []
+  if (s.memory >= 1) out.push('memory')
+  if (s.io >= 1) out.push('io')
+  if (lasted(m.hotAt, targets.hotMinutes, t)) out.push('lag')
+  return out
+}
+
+/**
+ * Where a new room goes: the active shared machine (not dedicated, not retiring) under its load
+ * target and its room ceiling with the least load, else the least loaded of them all with `full`
+ * (every one is over: time for another). Null when none is live. `exclude`: a machine to leave out.
+ */
+export function chooseMachine (machines, counts, t, targets = TARGETS, { exclude = null } = {}) {
+  const live = machines.filter((m) => !m.dedicatedRoom && m.id !== exclude && alive(m, t))
   if (!live.length) return null
   const placed = (m) => counts[m.id] || 0
-  const under = live.filter((m) => placed(m) < (m.capacity || 1))
+  const under = live.filter((m) => placed(m) < (m.capacity || 1) && !overReasons(m, targets, t).length)
   const pool = under.length ? under : live
-  const loaded = (m) => Number(m.load && m.load.rooms) || 0
-  const best = [...pool].sort((a, b) => (under.length ? loaded(a) - loaded(b) : 0) || placed(a) - placed(b) || a.id.localeCompare(b.id))[0]
+  const loaded = (m) => num(m.load && m.load.rooms)
+  const best = [...pool].sort((a, b) => loadScore(a, placed(a), targets) - loadScore(b, placed(b), targets) || loaded(a) - loaded(b) || placed(a) - placed(b) || a.id.localeCompare(b.id))[0]
   return { machine: best, full: !under.length }
 }
 
+/**
+ * The rooms of a machine that may move now, lightest first (bytes a minute, then connections,
+ * then size): placed (not mid-move), not moved in the last `moveGapMinutes`, and no git push or
+ * commit running in it (its machine's report says).
+ */
+export function movableRooms (rooms, t, targets = TARGETS) {
+  const s = (r) => r.stats || {}
+  return rooms
+    .filter((r) => r.state === 'placed' && !(r.movedAt && t - r.movedAt < targets.moveGapMinutes * MIN) && !(num(s(r).git) > 0))
+    .sort((a, b) => (num(s(a).bytesIn) + num(s(a).bytesOut)) - (num(s(b).bytesIn) + num(s(b).bytesOut)) || num(s(a).conns) - num(s(b).conns) || num(s(a).docBytes) - num(s(b).docBytes) || a.room.localeCompare(b.room))
+}
+
 /**
  * @param {object} o
  * @param {object} o.store      the API's store (relay machine and room methods)
  * @param {string} o.relayUrl   the relay's public address (http(s)://)
  * @param {string} o.relaySecret RELAY_API_SECRET
- * @param {object|null} [o.fly] relay-machines.js client, or null: then heavy rooms are only logged
+ * @param {object|null} [o.fly] relay-machines.js client, or null: then heavy rooms are only logged, and no machine is added or destroyed
  * @param {object} [o.limits]   what heavy is (relay-metrics.js HEAVY)
+ * @param {object} [o.targets]  the load a shared machine stays under (TARGETS)
  */
-export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch = globalThis.fetch, now = Date.now, log = () => {}, fly = null, limits = HEAVY, helloWaitMs = HELLO_WAIT_MS, pollMs = 2000, callTimeoutMs = 30 * 1000, moveTimeoutMs = 30 * 60 * 1000, volumeGb = 3 } = {}) {
+export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch = globalThis.fetch, now = Date.now, log = () => {}, fly = null, limits = HEAVY, targets = TARGETS, helloWaitMs = HELLO_WAIT_MS, pollMs = 2000, callTimeoutMs = 30 * 1000, moveTimeoutMs = 30 * 60 * 1000, volumeGb = 3, tickMs = TICK_MS, moveRetryMs = MOVE_RETRY_MS, retireGraceMs = RETIRE_GRACE_MS, maxDrainMoves = 20 } = {}) {
+  targets = { ...TARGETS, ...(targets || {}) }
   const base = String(relayUrl).replace(/\/+$/, '').replace(/^ws(s?):\/\//, 'http$1://')
   const jobs = new Set() // promises of jobs this API machine runs (tests wait on them)
   const told = new Set() // heavy rooms already logged as having no Fly to move to
+  const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+  let lastTick = -Infinity
 
   /** One relay machine: the relay's address, with Fly's proxy told which machine. */
   async function callRelay (machineId, method, path, body, { timeoutMs = callTimeoutMs } = {}) {
@@ -59,14 +173,14 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
     const text = await res.text()
     let out = null
     try { out = text ? JSON.parse(text) : null } catch { out = { text } }
-    if (!res.ok) throw new Error(`relay ${machineId || ''} ${path} answered ${res.status}${out && (out.error || out.text) ? `: ${out.error || out.text}` : ''}`)
+    if (!res.ok) throw Object.assign(new Error(`relay ${machineId || ''} ${path} answered ${res.status}${out && (out.error || out.text) ? `: ${out.error || out.text}` : ''}`), { status: res.status })
     return out
   }
 
-  /** Machines that are up: every one gets GitHub's nudges. */
+  /** Machines that are up (a retiring one too, while it holds rooms): every one gets GitHub's nudges. */
   async function liveMachines () {
     const t = now()
-    return (await store.relayMachines()).filter((m) => alive(m, t))
+    return (await store.relayMachines()).filter((m) => up(m, t))
   }
 
   /** A nudge for every machine (each tells the rooms it holds); one plain request while none has said hello. */
@@ -79,6 +193,15 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
     return ids.length
   }
 
+  /** A room landed on a retiring machine (on its disk, or nowhere else to go): it stays, and so does the machine. */
+  async function keepIfRetiring (machineId) {
+    const m = await store.relayMachine(machineId)
+    if (m && m.state === 'retiring') {
+      await store.putRelayMachine({ id: machineId, state: 'active', emptyAt: null })
+      log(`relay ${machineId} holds a room again: it is not retired`)
+    }
+  }
+
   /** A machine started: it is live, and the rooms on its disk are its own unless placed elsewhere (those it moves aside). */
   async function hello ({ machine, region = '', capacity, rooms = [] }) {
     const cur = await store.relayMachine(machine)
@@ -103,32 +226,47 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
     if (!(await store.relayMachine(machine))) await store.putRelayMachine({ id: machine, state: 'active', helloAt: now(), reportedAt: now() })
     let target = machine
     if (!have) {
-      const pick = chooseMachine(await store.relayMachines(), await store.relayRoomCounts(), now())
+      const pick = chooseMachine(await store.relayMachines(), await store.relayRoomCounts(), now(), targets)
       if (pick) {
         target = pick.machine.id
-        if (pick.full) startJob('scale', { region: pick.machine.region }, () => scaleUp(pick.machine))
+        if (pick.full) scaleOnce(pick.machine)
       }
     }
     const r = await store.placeRelayRoom(room, target)
+    await keepIfRetiring(r.machineId)
     return { machine: r.machineId, state: r.state }
   }
 
-  /** A machine's minute: its load, and each room's traffic and size; heavy rooms get a machine of their own. */
+  /** A machine's minute: its load, and each room's traffic and size; heavy rooms get a machine of their own; then the cluster is looked over. */
   async function report ({ machine, load = {}, rooms = [] }) {
     const t = now()
-    const m = await store.putRelayMachine({ id: machine, load, reportedAt: t, state: 'active' })
+    const prev = await store.relayMachine(machine)
+    const placedHere = (await store.relayRoomsOn(machine)).length
+    const hotAt = hotMinute(load, targets) ? (prev && prev.hotAt) || t : null
+    const over = overReasons({ load, hotAt }, targets, t)
+    const empty = !placedHere && !(num(load.rooms) > 0)
+    const m = await store.putRelayMachine({
+      id: machine,
+      load,
+      reportedAt: t,
+      // A retiring machine stays retiring while it reports; any other that reports is up.
+      state: prev && prev.state === 'retiring' ? 'retiring' : 'active',
+      hotAt,
+      overAt: over.length ? (prev && prev.overAt) || t : null,
+      emptyAt: empty ? (prev && prev.emptyAt) || t : null
+    })
     let heavy = 0
     for (const s of rooms) {
       if (!s || !ROOM.test(String(s.room))) continue
       const cur = await store.relayRoom(s.room)
       if (!cur || cur.machineId !== machine) continue
-      const stats = { conns: s.conns | 0, msgsIn: s.msgsIn | 0, msgsOut: s.msgsOut | 0, bytesIn: +s.bytesIn || 0, bytesOut: +s.bytesOut || 0, docBytes: +s.docBytes || 0, diskBytes: +s.diskBytes || 0 }
+      const stats = { conns: s.conns | 0, msgsIn: s.msgsIn | 0, msgsOut: s.msgsOut | 0, bytesIn: +s.bytesIn || 0, bytesOut: +s.bytesOut || 0, docBytes: +s.docBytes || 0, diskBytes: +s.diskBytes || 0, git: s.git | 0 }
       const busy = busyMinute(stats, limits) ? (cur.busyMinutes || 0) + 1 : 0
       const why = heavyReasons(stats, limits, busy)
       await store.setRelayRoom(s.room, { stats, busyMinutes: busy, heavy: why, reportedAt: t })
       if (why.length && m.dedicatedRoom !== s.room && cur.state === 'placed') { heavy++; onHeavy(s.room, m, why) }
     }
-    // A dedicated machine whose room is gone (ended, or swept) takes other rooms again.
+    // A dedicated machine whose room is gone (ended, or swept) takes other rooms again (and, left empty, is retired).
     if (m.dedicatedRoom) {
       const r = await store.relayRoom(m.dedicatedRoom)
       const job = await store.relayJob(`dedicate:${m.dedicatedRoom}`)
@@ -137,7 +275,9 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
         log(`relay ${machine}: ${m.dedicatedRoom} is not on it any more; it takes other rooms now`)
       }
     }
-    return { ok: true, heavy }
+    if (over.length && !(prev && prev.overAt)) log(`relay ${machine} is over its load target (${over.join(', ')})`)
+    tick()
+    return { ok: true, heavy, over }
   }
 
   async function forget ({ room, machine }) {
@@ -145,10 +285,10 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
     return { ok: true }
   }
 
-  /** Runs `fn` as the job `key` unless it is running (here or on another API machine) or ran just now. */
-  function startJob (key, detail, fn) {
-    const p = (async () => {
-      if (!(await store.takeRelayJob(key, { detail, staleMs: JOB_STALE_MS, retryMs: JOB_RETRY_MS }))) return null
+  /** Runs `fn` as the job `key` unless it is running (here or on another API machine) or ran within `retryMs`. */
+  function startJob (key, detail, fn, { retryMs = JOB_RETRY_MS } = {}) {
+    return track((async () => {
+      if (!(await store.takeRelayJob(key, { detail, staleMs: JOB_STALE_MS, retryMs }))) return null
       try {
         const out = await fn()
         await store.finishRelayJob(key, { state: 'done', detail: { ...detail, ...(out || {}) } })
@@ -158,7 +298,10 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
         await store.finishRelayJob(key, { state: 'failed', detail: { ...detail, error: String(err.message).slice(0, 300) } }).catch(() => {})
         return null
       }
-    })().catch((err) => { log(`relay job ${key}: ${err.message}`); return null })
+    })().catch((err) => { log(`relay job ${key}: ${err.message}`); return null }))
+  }
+
+  function track (p) {
     jobs.add(p)
     p.finally(() => jobs.delete(p))
     return p
@@ -177,32 +320,42 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
     const others = (await store.relayRoomsOn(from.id)).filter((r) => r.room !== room)
     if (!others.length) {
       // Alone on its machine already: that machine is now its own, and takes no others.
-      await store.putRelayMachine({ id: from.id, dedicatedRoom: room })
+      await store.putRelayMachine({ id: from.id, dedicatedRoom: room, state: 'active', emptyAt: null })
       log(`room ${room} keeps relay ${from.id} to itself`)
       return { machine: from.id, moved: false }
     }
     // A machine already up with no rooms (one an earlier, failed move made, or a spare shared one)
     // goes first: a retry never leaves the last try's machine running empty.
     const spare = await spareMachine(room, from.id)
-    if (spare) await store.putRelayMachine({ id: spare, dedicatedRoom: room })
+    if (spare) await store.putRelayMachine({ id: spare, dedicatedRoom: room, emptyAt: null })
     const made = spare || await newMachine({ region: from.region, dedicatedRoom: room, template: from.id })
-    await moveRoom(room, from.id, made)
+    const r = await moveRoom(room, from.id, made)
+    if (!r.moved) throw new Error(`room ${room} could not move now (${r.why})`)
     return { machine: made, moved: true }
   }
 
-  /** A live machine other than `fromId` holding no rooms, shared or kept for `room`; null when none. */
+  /** An active machine other than `fromId` holding no rooms, shared or kept for `room`; null when none. */
   async function spareMachine (room, fromId) {
-    for (const m of await liveMachines()) {
-      if (m.id === fromId || (m.dedicatedRoom && m.dedicatedRoom !== room)) continue
+    const t = now()
+    for (const m of await store.relayMachines()) {
+      if (!alive(m, t) || m.id === fromId || (m.dedicatedRoom && m.dedicatedRoom !== room)) continue
       if (!(await store.relayRoomsOn(m.id)).length) return m.id
     }
     return null
   }
 
-  /** Every shared machine is at capacity: one more shared machine. */
+  /** Every shared machine is over its target: one more shared machine (one at a time, the 'scale' job). */
+  function scaleOnce (like) {
+    if (!fly) {
+      if (!told.has(':scale')) { told.add(':scale'); log('every shared relay machine is over its load target; no Fly token, so no machine is added') }
+      return null
+    }
+    return startJob('scale', { region: like.region }, () => scaleUp(like))
+  }
+
   async function scaleUp (like) {
     const id = await newMachine({ region: like.region, dedicatedRoom: null, template: like.id })
-    log(`added relay ${id}: every shared machine was at capacity`)
+    log(`added relay ${id}: every shared machine was over its load target`)
     return { machine: id }
   }
 
@@ -215,22 +368,33 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
       const row = await store.relayMachine(m.id)
       if (row && row.helloAt) return m.id
       if (now() >= until) throw new Error(`relay machine ${m.id} did not start in ${Math.round(helloWaitMs / 1000)} s`)
-      await new Promise((resolve) => setTimeout(resolve, pollMs))
+      await sleep(pollMs)
     }
   }
 
   /**
    * Moves `room` from machine `from` to `to`: the source lets it go (its people reconnect), the
    * target copies its files from the source, placement flips, the source forgets its copy. Until
-   * the flip, a failure puts the room back where it was.
+   * the flip, a failure puts the room back where it was. { moved: false, why } when the source
+   * would not let it go now (a git push or commit running in it) or the target stopped taking rooms.
    */
   async function moveRoom (room, from, to) {
     await store.setRelayRoom(room, { state: 'moving' })
-    let flipped = false
     try {
       await callRelay(from, 'POST', `/cluster/rooms/${room}/release`, {})
+    } catch (err) {
+      await store.setRelayRoom(room, { machineId: from, state: 'placed' }).catch(() => {})
+      if (err.status === 409) { log(`room ${room} stays on relay ${from} for now: ${err.message}`); return { moved: false, why: 'busy' } }
+      await callRelay(from, 'POST', `/cluster/rooms/${room}/restore`, {}).catch((e) => log(`could not restore ${room} on ${from}: ${e.message}`))
+      throw err
+    }
+    let flipped = false
+    try {
       const got = await callRelay(to, 'POST', `/cluster/rooms/${room}/fetch`, { from }, { timeoutMs: moveTimeoutMs })
-      await store.setRelayRoom(room, { machineId: to, state: 'placed', busyMinutes: 0 })
+      // The target may have started retiring while the copy ran: then the room stays where it was.
+      const target = await store.relayMachine(to)
+      if (!target || target.state !== 'active') throw Object.assign(new Error(`relay ${to} is ${target ? target.state : 'gone'}`), { notTaking: true })
+      await store.setRelayRoom(room, { machineId: to, state: 'placed', busyMinutes: 0, movedAt: now() })
       flipped = true
       log(`moved room ${room} from relay ${from} to ${to} (${got && got.files} files, ${got && got.bytes} bytes)`)
     } catch (err) {
@@ -239,13 +403,143 @@ export function relayCluster ({ store, relayUrl = '', relaySecret = '', fetch =
         await callRelay(from, 'POST', `/cluster/rooms/${room}/restore`, {}).catch((e) => log(`could not restore ${room} on ${from}: ${e.message}`))
         await callRelay(to, 'POST', `/cluster/rooms/${room}/forget`, { to: from }).catch(() => {})
       }
+      if (err.notTaking) { log(`room ${room} stays on relay ${from}: ${err.message}`); return { moved: false, why: 'target' } }
       throw err
     }
     await callRelay(from, 'POST', `/cluster/rooms/${room}/forget`, { to }).catch((err) => log(`relay ${from} could not delete its copy of ${room}: ${err.message}`))
+    return { moved: true }
+  }
+
+  // ---- scaling both ways, on load ----
+
+  /** At most every `tickMs` (per API machine), on a machine's report: the whole cluster is looked over. */
+  function tick () {
+    const t = now()
+    if (t - lastTick < tickMs) return null
+    lastTick = t
+    return track(autoscale().catch((err) => { log(`relay autoscale: ${err.message}`); return null }))
+  }
+
+  async function autoscale () {
+    const t = now()
+    const machines = await store.relayMachines()
+    const counts = await store.relayRoomCounts()
+    const placed = (m) => counts[m.id] || 0
+    const shared = machines.filter((m) => !m.dedicatedRoom && up(m, t))
+    const active = shared.filter((m) => m.state === 'active')
+    // Shared machines staying: retiring one needs this to stay at least the minimum.
+    let keep = active.length
+
+    // A retiring machine still holding rooms hands them on, one at a time.
+    for (const m of shared) if (m.state === 'retiring' && placed(m)) startJob(`move:${m.id}`, { machine: m.id, why: 'retiring' }, () => drain(m.id), { retryMs: moveRetryMs })
+
+    // Empty long enough: stopped and destroyed with its volume, keeping the minimum.
+    for (const m of shared) {
+      if (placed(m) || num(m.load && m.load.rooms) > 0 || !m.emptyAt || t - m.emptyAt < targets.emptyMinutes * MIN) continue
+      if (!fly) {
+        if (!told.has(`:empty:${m.id}`)) { told.add(`:empty:${m.id}`); log(`relay ${m.id} has been empty for ${targets.emptyMinutes} minutes; no Fly token, so it keeps running`) }
+        continue
+      }
+      if (m.state === 'active') {
+        if (keep - 1 < targets.minMachines) continue
+        keep--
+      } else if (keep < targets.minMachines) {
+        await store.putRelayMachine({ id: m.id, state: 'active' })
+        keep++
+        continue
+      }
+      startJob(`retire:${m.id}`, { machine: m.id }, () => retire(m.id))
+    }
+
+    // Over target for a while, with rooms to spare: its lightest rooms move off, one at a time.
+    for (const m of active) {
+      if (placed(m) < 2 || !lasted(m.overAt, targets.rebalanceMinutes, t)) continue
+      startJob(`move:${m.id}`, { machine: m.id, why: 'over target' }, () => rebalance(m.id), { retryMs: moveRetryMs })
+    }
+
+    // Two (or more) lightly loaded machines that have been up a while: the emptier hands its rooms
+    // to the others and, once empty, is retired. One at a time, never below the minimum.
+    if (fly && !shared.some((m) => m.state === 'retiring') && active.length > Math.max(1, targets.minMachines)) {
+      const light = active.filter((m) => placed(m) > 0 && !m.overAt && (m.helloAt || 0) <= t - targets.emptyMinutes * MIN && loadScore(m, placed(m), targets) < targets.consolidateBelow)
+      if (light.length >= 2) {
+        const [from] = light.sort((a, b) => placed(a) - placed(b) || loadScore(a, placed(a), targets) - loadScore(b, placed(b), targets) || (b.createdAt || 0) - (a.createdAt || 0) || b.id.localeCompare(a.id))
+        await store.putRelayMachine({ id: from.id, state: 'retiring' })
+        log(`relay ${from.id} is lightly loaded, and so is another: its ${placed(from)} room(s) move off and it is retired`)
+        startJob(`move:${from.id}`, { machine: from.id, why: 'consolidating' }, () => drain(from.id), { retryMs: moveRetryMs })
+      }
+    }
+  }
+
+  /** A machine over its target: its lightest movable room goes to the least loaded machine under target, else a new machine is asked for. */
+  async function rebalance (id) {
+    const m = await store.relayMachine(id)
+    if (!m || m.state !== 'active' || m.dedicatedRoom || !lasted(m.overAt, targets.rebalanceMinutes, now())) return { skipped: 'not over target' }
+    const all = await store.relayRoomsOn(id)
+    const rooms = movableRooms(all, now(), targets).filter((r) => !(r.heavy && r.heavy.length))
+    if (all.length < 2 || !rooms.length) return { skipped: 'no room may move now' }
+    const pick = chooseMachine(await store.relayMachines(), await store.relayRoomCounts(), now(), targets, { exclude: id })
+    if (!pick || pick.full) {
+      scaleOnce(m)
+      return { waiting: 'a machine under target' }
+    }
+    for (const r of rooms) {
+      const out = await moveRoom(r.room, id, pick.machine.id)
+      if (out.moved) return { room: r.room, to: pick.machine.id }
+      if (out.why === 'target') break
+    }
+    return { skipped: 'no room could move now' }
+  }
+
+  /** A retiring machine's rooms go to the least loaded machines under target; with none, it is not retired after all. */
+  async function drain (id) {
+    const moved = []
+    const skip = new Set()
+    for (let i = 0; i < maxDrainMoves; i++) {
+      const m = await store.relayMachine(id)
+      if (!m || m.state !== 'retiring') break
+      const rooms = movableRooms(await store.relayRoomsOn(id), now(), targets).filter((r) => !skip.has(r.room))
+      if (!rooms.length) break
+      const pick = chooseMachine(await store.relayMachines(), await store.relayRoomCounts(), now(), targets, { exclude: id })
+      if (!pick || pick.full) {
+        await store.putRelayMachine({ id, state: 'active' })
+        log(`relay ${id} keeps its rooms: no other machine is under its load target`)
+        return { moved, cancelled: true }
+      }
+      const out = await moveRoom(rooms[0].room, id, pick.machine.id)
+      if (out.moved) moved.push(rooms[0].room); else skip.add(rooms[0].room)
+    }
+    return { moved }
+  }
+
+  /** An empty shared machine: retiring (no new rooms), then stopped and destroyed with its volume, unless a room lands on it first. */
+  async function retire (id) {
+    const m = await store.relayMachine(id)
+    if (!m || m.dedicatedRoom || !up(m, now())) return { skipped: 'not a live shared machine' }
+    if ((await store.relayRoomsOn(id)).length) return { skipped: 'holds rooms' }
+    const others = (await store.relayMachines()).filter((x) => x.id !== id && !x.dedicatedRoom && alive(x, now()))
+    if (others.length < targets.minMachines) {
+      if (m.state === 'retiring') await store.putRelayMachine({ id, state: 'active' })
+      return { skipped: 'the last machines are kept' }
+    }
+    if (m.state !== 'retiring') await store.putRelayMachine({ id, state: 'retiring' })
+    if (retireGraceMs > 0) await sleep(retireGraceMs)
+    const cur = await store.relayMachine(id)
+    if (!cur || cur.state !== 'retiring' || (await store.relayRoomsOn(id)).length) {
+      if (cur && cur.state === 'retiring') await store.putRelayMachine({ id, state: 'active', emptyAt: null })
+      log(`relay ${id} is not retired: a room landed on it`)
+      return { cancelled: true }
+    }
+    const out = await fly.destroyRelayMachine(id)
+    await store.putRelayMachine({ id, state: 'gone', emptyAt: null, hotAt: null, overAt: null })
+    // A room placed on it in the last moment had nothing there yet: it is placed afresh when next asked for.
+    for (const r of await store.relayRoomsOn(id)) { await store.forgetRelayRoom(r.room, id); log(`room ${r.room} was placed on relay ${id} as it was destroyed; it is placed again when next asked for`) }
+    log(`retired relay ${id}: empty for ${targets.emptyMinutes} minutes (destroyed with ${(out && out.volumes && out.volumes.length) || 0} volume(s))`)
+    for (const v of (out && out.leftover) || []) log(`relay ${id} is destroyed, but Fly would not delete its volume ${v.volume} (${v.error}): delete it by hand`)
+    return { destroyed: true, volumes: (out && out.volumes) || [], ...(out && out.leftover && out.leftover.length ? { leftover: out.leftover.map((v) => v.volume) } : {}) }
   }
 
   /** Waits for every job started so far (tests). */
   async function idle () { while (jobs.size) await Promise.allSettled([...jobs]) }
 
-  return { hello, place, report, forget, tellRelays, callRelay, moveRoom, dedicate, idle, valid: { room: (r) => ROOM.test(String(r)), machine: (m) => MACHINE.test(String(m)) } }
+  return { hello, place, report, forget, tellRelays, callRelay, moveRoom, dedicate, autoscale, rebalance, drain, retire, idle, targets, valid: { room: (r) => ROOM.test(String(r)), machine: (m) => MACHINE.test(String(m)) } }
 }

package/src/api/relay-machines.js

@@ -1,6 +1,7 @@
 // Fly's Machines API, for the relay's machines (src/api/relay-cluster.js): a new relay machine is
 // a copy of a running one's config (image, size, services, checks, env), with a volume of its own
 // at the same path. `fly deploy` updates it with the others: it keeps the app's process group.
+// A machine the cluster no longer needs (empty a while) is stopped and destroyed with its volume.
 // Docs: https://fly.io/docs/machines/api/
 export const FLY_API = 'https://api.machines.dev/v1'
 
@@ -14,7 +15,7 @@ export function relayFlyFromEnv (env = process.env, o = {}) {
   return relayFlyMachines({ token, app: env.QUILT_RELAY_APP || 'cowove-relay', ...o })
 }
 
-export function relayFlyMachines ({ token, app, fetch = globalThis.fetch, base = FLY_API, timeoutMs = 60 * 1000 }) {
+export function relayFlyMachines ({ token, app, fetch = globalThis.fetch, base = FLY_API, timeoutMs = 60 * 1000, volumeWaitMs = 2 * 60 * 1000, pollMs = 3000 }) {
   const root = String(base).replace(/\/+$/, '')
   const call = async (method, path, body) => {
     const res = await fetch(`${root}${path}`, {
@@ -26,7 +27,7 @@ export function relayFlyMachines ({ token, app, fetch = globalThis.fetch, base =
     const text = await res.text()
     let out = null
     try { out = text ? JSON.parse(text) : null } catch { out = { error: text } }
-    if (!res.ok) throw new Error(`Fly answered ${res.status} for ${method} ${path.split('?')[0]}${out && out.error ? `: ${String(out.error).slice(0, 200)}` : ''}`)
+    if (!res.ok) throw Object.assign(new Error(`Fly answered ${res.status} for ${method} ${path.split('?')[0]}${out && out.error ? `: ${String(out.error).slice(0, 200)}` : ''}`), { status: res.status })
     return out
   }
   const appPath = `/apps/${encodeURIComponent(app)}`
@@ -56,5 +57,36 @@ export function relayFlyMachines ({ token, app, fetch = globalThis.fetch, base =
     return { id: m.id, region: m.region || where, volume: vol.id }
   }
 
-  return { createRelayMachine, app }
+  /**
+   * Stops relay machine `id`, destroys it, then deletes its volumes (Fly frees a volume a moment after
+   * its machine goes, so that is tried until `volumeWaitMs`). A machine Fly no longer has is done
+   * already. Resolves { id, volumes, leftover }: the volumes deleted, and any Fly would not delete
+   * in time (the machine is gone either way; those are for someone to delete by hand).
+   */
+  async function destroyRelayMachine (id) {
+    const mp = `${appPath}/machines/${encodeURIComponent(id)}`
+    let m = null
+    try { m = await call('GET', mp) } catch (err) { if (err.status !== 404) throw err }
+    const volumes = ((m && m.config && Array.isArray(m.config.mounts) && m.config.mounts) || []).map((x) => x && x.volume).filter(Boolean)
+    if (m) {
+      await call('POST', `${mp}/stop`, {}).catch(() => {})
+      await call('GET', `${mp}/wait?state=stopped&timeout=60`).catch(() => {})
+      try { await call('DELETE', `${mp}?force=true`) } catch (err) { if (err.status !== 404) throw err }
+    }
+    const until = Date.now() + volumeWaitMs
+    const deleted = []
+    const leftover = []
+    for (const v of volumes) {
+      for (;;) {
+        try { await call('DELETE', `${appPath}/volumes/${encodeURIComponent(v)}`); deleted.push(v); break } catch (err) {
+          if (err.status === 404) { deleted.push(v); break }
+          if (Date.now() >= until) { leftover.push({ volume: v, error: err.message }); break }
+          await new Promise((resolve) => setTimeout(resolve, pollMs))
+        }
+      }
+    }
+    return { id, volumes: deleted, leftover }
+  }
+
+  return { createRelayMachine, destroyRelayMachine, app }
 }