heyquilt 0.0.1 → 0.3.90

package/src/relay-cluster.js

@@ -0,0 +1,454 @@
+// The relay on many machines (docs/superpowers/specs/2026-10-17-relay-scaling.md). Each room lives
+// on one machine; the accounts API says which (src/api/relay-cluster.js). A machine that gets a
+// request for a room it does not hold answers `fly-replay: instance=<machine>` and Fly's proxy
+// sends the request there instead; a body too big for Fly to replay (over 1 MB, or of unknown
+// length) is streamed to that machine over Fly's private network. The API reaches one machine
+// with `fly-force-instance-id`. Machines say hello when they start and report every minute.
+//
+// Off unless QUILT_CLUSTER=1: one machine, as before. On, it needs the machine id (FLY_MACHINE_ID),
+// the accounts API (QUILT_API_URL, RELAY_API_SECRET), sign-in, and with git sync the one file key
+// every machine shares (QUILT_RELAY_FILE_KEY).
+import fs from 'node:fs'
+import path from 'node:path'
+import http from 'node:http'
+import https from 'node:https'
+
+// A room placed elsewhere is asked about again after this; one placed here stays here until the API moves it.
+export const PLACEMENT_TTL_MS = 30 * 1000
+// Fly's proxy replays a request's body up to this size.
+export const REPLAY_MAX_BODY = 1024 * 1024
+export const REPORT_MS = 60 * 1000
+// A hosted agent's sessions and webhook from the API, kept this long.
+export const INDEX_TTL_MS = 30 * 1000
+export const SAME_OWNER_TTL_MS = 5 * 60 * 1000
+const API_TIMEOUT_MS = 10 * 1000
+const ROOM_RE = /^[A-Za-z0-9_-]{1,64}$/
+const MACHINE_RE = /^[A-Za-z0-9_-]{1,64}$/
+// Files in the data folder that look like rooms (`<name>.json`) and aren't.
+export const NOT_ROOMS = new Set(['agent-links', 'hosted-agents', 'leases'])
+export const MOVING = 'This session is moving to another relay machine; try again in a moment'
+export const ELSEWHERE = 'This session lives on another relay machine; try again in a moment'
+export const NO_PLACEMENT = "The relay can't find which machine holds this session right now; try again in a moment"
+
+/** Cluster settings, from options or the environment (Fly sets FLY_MACHINE_ID, FLY_REGION, FLY_APP_NAME). */
+export function clusterConfig (opts = {}, env = process.env) {
+  const on = opts.cluster ?? /^(1|true|yes)$/i.test(String(env.QUILT_CLUSTER || ''))
+  const app = env.FLY_APP_NAME || ''
+  const port = env.PORT || '4321'
+  const capacity = Number(opts.capacity ?? env.QUILT_RELAY_CAPACITY ?? 50)
+  return {
+    on: !!on,
+    machineId: String(opts.machineId ?? env.FLY_MACHINE_ID ?? ''),
+    region: String(opts.region ?? env.FLY_REGION ?? ''),
+    capacity: Number.isInteger(capacity) && capacity > 0 ? capacity : 50,
+    // Another machine of this app, over Fly's private network; with no {id} in it, the address is
+    // used as it is, with fly-force-instance-id (a proxy that honours it, as tests use).
+    peerUrl: String(opts.peerUrl ?? env.QUILT_PEER_URL ?? (app ? `http://{id}.vm.${app}.internal:${port}` : '')),
+    reportMs: Number(opts.reportMs ?? env.QUILT_CLUSTER_REPORT_MS) > 0 ? Number(opts.reportMs ?? env.QUILT_CLUSTER_REPORT_MS) : REPORT_MS,
+    placementTtlMs: Number(opts.placementTtlMs) > 0 ? Number(opts.placementTtlMs) : PLACEMENT_TTL_MS
+  }
+}
+
+/** What cluster mode is missing; [] when it can run. */
+export function clusterProblems (cc, cfg, { fileKey = '' } = {}) {
+  if (!cc.on) return []
+  const out = []
+  if (!MACHINE_RE.test(cc.machineId)) out.push('FLY_MACHINE_ID (this machine\'s id)')
+  if (!cfg.apiUrl) out.push('QUILT_API_URL')
+  if (!cfg.relayApiSecret) out.push('RELAY_API_SECRET')
+  if (!cfg.passPublicKey) out.push('QUILT_PASS_PUBLIC_KEY (sign-in)')
+  if (cfg.gitSync && !fileKey) out.push('QUILT_RELAY_FILE_KEY (the file key every machine shares)')
+  if (!cc.peerUrl) out.push('QUILT_PEER_URL or FLY_APP_NAME (how machines reach each other)')
+  return out
+}
+
+/** The rooms kept in a data folder (their `.json` or `.ydoc`). */
+export function storedRooms (dataDir) {
+  if (!dataDir) return []
+  const out = new Set()
+  let files = []
+  try { files = fs.readdirSync(dataDir) } catch { return [] }
+  for (const f of files) {
+    const m = /^(.+)\.(json|ydoc)$/.exec(f)
+    if (m && ROOM_RE.test(m[1]) && !NOT_ROOMS.has(m[1])) out.add(m[1])
+  }
+  return [...out].sort()
+}
+
+/** The room a request's path is about ('' for none): git, chat files, large files, chat links. */
+export function roomOfPath (pathname) {
+  const m = /^\/git\/([A-Za-z0-9_-]{1,64})\.git\//.exec(pathname) ||
+    /^\/files\/([A-Za-z0-9_-]{1,64})(?:\/|$)/.exec(pathname) ||
+    /^\/blobs\/([A-Za-z0-9_-]{1,64})\//.exec(pathname) ||
+    /^\/c\/([A-Za-z0-9_-]{1,64})\//.exec(pathname)
+  return m ? m[1] : ''
+}
+
+/** The machine a request was already sent on from (Fly's replay, or another machine's forward), or ''. */
+export function replaySource (req) {
+  const fly = /(?:^|;)\s*instance=([A-Za-z0-9_-]+)/.exec(String(req.headers['fly-replay-src'] || ''))
+  if (fly) return fly[1]
+  const fwd = String(req.headers['x-quilt-forwarded'] || '')
+  return MACHINE_RE.test(fwd) ? fwd : ''
+}
+
+/** Whether Fly can replay this request (its body, if any, is known to be at most 1 MB). */
+export function canReplay (req, readBody = null) {
+  if (readBody) return readBody.length <= REPLAY_MAX_BODY
+  if (req.method === 'GET' || req.method === 'HEAD') return true
+  if (req.headers['transfer-encoding']) return false
+  const n = Number(req.headers['content-length'] || 0)
+  return Number.isFinite(n) && n <= REPLAY_MAX_BODY
+}
+
+export class Cluster {
+  constructor ({ cc, apiUrl, secret, fetch = globalThis.fetch, log = () => {}, dataDir = null, now = Date.now }) {
+    this.cc = cc
+    this.machineId = cc.machineId
+    this.api = String(apiUrl).replace(/\/+$/, '')
+    this.secret = secret
+    this.fetch = fetch
+    this.log = log
+    this.dataDir = dataDir
+    this.now = now
+    this.here = new Set() // rooms the API placed on this machine
+    this.elsewhere = new Map() // room -> { machine, at }
+    this.moving = new Set() // rooms being moved off this machine: refused until forgotten or restored
+    this.hosted = new Map() // account -> { rooms, webhook, at }
+    this.owners = new Map() // `${room}\n${account}` -> { same, at }
+    this.hello = null // the promise of this start's hello
+  }
+
+  async call (method, p, body) {
+    const res = await this.fetch(`${this.api}${p}`, {
+      method,
+      headers: { 'content-type': 'application/json', authorization: `Bearer ${this.secret}` },
+      ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+      signal: AbortSignal.timeout(API_TIMEOUT_MS)
+    })
+    if (!res.ok) throw new Error(`the accounts API answered ${res.status}`)
+    return res.json()
+  }
+
+  onDisk (room) {
+    if (!this.dataDir) return false
+    return fs.existsSync(path.join(this.dataDir, `${room}.json`)) || fs.existsSync(path.join(this.dataDir, `${room}.ydoc`))
+  }
+
+  /** Says hello (tried again every 30 s until the API answers): the rooms on this disk; returns those placed elsewhere. */
+  start ({ onDisplaced = () => {}, retryMs = 30 * 1000 } = {}) {
+    let timer = null
+    const once = async () => {
+      const rooms = storedRooms(this.dataDir)
+      try {
+        const r = await this.call('POST', '/v1/relay/machines/hello', { machine: this.machineId, region: this.cc.region, capacity: this.cc.capacity, rooms })
+        const displaced = new Set(Array.isArray(r.displaced) ? r.displaced : [])
+        for (const room of rooms) if (!displaced.has(room)) this.here.add(room)
+        if (displaced.size) onDisplaced([...displaced])
+        this.log(`cluster: machine ${this.machineId} holds ${rooms.length - displaced.size} stored room(s)`)
+        return true
+      } catch (err) {
+        this.log(`cluster: could not say hello to the accounts API (${err.message}); trying again in ${Math.round(retryMs / 1000)} s`)
+        await new Promise((resolve) => { timer = setTimeout(resolve, retryMs); timer.unref?.() })
+        return this.closed ? false : once()
+      }
+    }
+    this.hello = once()
+    this.stopHello = () => clearTimeout(timer)
+    return this.hello
+  }
+
+  stop () { this.closed = true; if (this.stopHello) this.stopHello() }
+
+  /** This machine's minute, for the API's placement and heavy rooms. */
+  async report (load, rooms) {
+    return this.call('POST', '/v1/relay/machines/report', { machine: this.machineId, load, rooms })
+  }
+
+  /** Whether this machine holds `room` (and may load it). */
+  holds (room) { return this.here.has(room) && !this.moving.has(room) }
+
+  /**
+   * Where `room` lives: { here } | { machine } | { moving } | { none } | { error }. `fresh` skips
+   * the cache of rooms placed elsewhere (a request replayed here that this machine doesn't hold);
+   * without `create` (nobody signed in asked) a room placed nowhere is { none } and stays unplaced.
+   */
+  async locate (room, { fresh = false, create = true } = {}) {
+    if (this.moving.has(room)) return { moving: true }
+    if (this.here.has(room)) return { here: true }
+    const c = this.elsewhere.get(room)
+    if (!fresh && c && this.now() - c.at < this.cc.placementTtlMs) return { machine: c.machine }
+    const have = this.onDisk(room)
+    let r
+    try {
+      r = await this.call('POST', '/v1/relay/placement', { room, machine: this.machineId, have, create })
+    } catch (err) {
+      // The API is down: what is on this disk is this machine's (moves delete the old copy); nothing else is.
+      if (have) { this.log(`cluster: placement unavailable (${err.message}); serving ${room} from this disk`); this.here.add(room); return { here: true } }
+      this.log(`cluster: placement unavailable for ${room} (${err.message})`)
+      return { error: NO_PLACEMENT }
+    }
+    if (r.state === 'moving') return { moving: true }
+    if (r.machine === null && r.state === 'none') return { none: true }
+    if (r.machine === this.machineId) { this.here.add(room); this.elsewhere.delete(room); return { here: true } }
+    if (!MACHINE_RE.test(String(r.machine || ''))) return { error: NO_PLACEMENT }
+    this.elsewhere.set(room, { machine: r.machine, at: this.now() })
+    if (this.elsewhere.size > 10000) this.elsewhere.delete(this.elsewhere.keys().next().value)
+    return { machine: r.machine }
+  }
+
+  /** The room is leaving this machine: refused from now on. */
+  release (room) { this.moving.add(room); this.here.delete(room) }
+  /** The move failed: the room is this machine's again. */
+  restore (room) { this.moving.delete(room); this.here.add(room) }
+  /** The room lives on `to` now (or nowhere: deleted). */
+  forget (room, to = '') {
+    this.moving.delete(room)
+    this.here.delete(room)
+    if (to && MACHINE_RE.test(to)) this.elsewhere.set(room, { machine: to, at: this.now() }); else this.elsewhere.delete(room)
+  }
+
+  /** A room deleted for good (its owner ended it and it was swept, or idle past its days): no longer placed. */
+  forgetPlacement (room) {
+    this.forget(room)
+    this.call('POST', '/v1/relay/placement/forget', { room, machine: this.machineId }).catch((err) => this.log(`cluster: could not forget ${room}'s placement: ${err.message}`))
+  }
+
+  /** How to reach machine `id`: { url, headers }. */
+  peer (id) {
+    const t = this.cc.peerUrl.replace(/\/+$/, '')
+    return t.includes('{id}') ? { url: t.replaceAll('{id}', id), headers: {} } : { url: t, headers: { 'fly-force-instance-id': id } }
+  }
+
+  /** Streams a request on to machine `id` (`body`: already read, else the request's own stream), and its answer back. */
+  forward (req, res, id, body = null) {
+    const p = this.peer(id)
+    const u = new URL(req.url, p.url)
+    const lib = u.protocol === 'https:' ? https : http
+    const headers = { ...req.headers, ...p.headers, 'x-quilt-forwarded': this.machineId, host: u.host }
+    if (body) { delete headers['transfer-encoding']; headers['content-length'] = String(body.length) }
+    const out = lib.request(u, { method: req.method, headers }, (back) => {
+      res.writeHead(back.statusCode || 502, back.headers)
+      back.pipe(res)
+    })
+    out.on('error', (err) => {
+      this.log(`cluster: could not reach machine ${id}: ${err.message}`)
+      if (!res.headersSent) { res.writeHead(503, { 'content-type': 'text/plain; charset=utf-8', 'retry-after': '5' }); res.end(ELSEWHERE) } else res.destroy()
+    })
+    if (body) out.end(body); else req.pipe(out)
+  }
+
+  /** Another machine's own data (an export), with the relay secret. */
+  fromPeer (id, p) {
+    const peer = this.peer(id)
+    const u = new URL(p, peer.url)
+    const lib = u.protocol === 'https:' ? https : http
+    return new Promise((resolve, reject) => {
+      const r = lib.request(u, { method: 'GET', headers: { ...peer.headers, authorization: `Bearer ${this.secret}` } }, (res) => {
+        if (res.statusCode !== 200) { res.resume(); return reject(new Error(`machine ${id} answered ${res.statusCode}`)) }
+        resolve(res)
+      })
+      r.on('error', reject)
+      r.end()
+    })
+  }
+
+  // ---- hosted agents across machines: the account's sessions and webhook, kept by the API ----
+
+  /** { rooms, webhook } for `account` from the API (cached), or null when it can't be asked. */
+  async hostedOf (account, { fresh = false } = {}) {
+    const c = this.hosted.get(account)
+    if (!fresh && c && this.now() - c.at < INDEX_TTL_MS) return c
+    try {
+      const r = await this.call('GET', `/v1/relay/hosted/${encodeURIComponent(account)}`)
+      const v = { rooms: Array.isArray(r.rooms) ? r.rooms.filter((x) => ROOM_RE.test(x)) : [], webhook: r.webhook || null, at: this.now() }
+      this.hosted.set(account, v)
+      if (this.hosted.size > 10000) this.hosted.delete(this.hosted.keys().next().value)
+      return v
+    } catch (err) {
+      this.log(`cluster: could not read ${account}'s sessions (${err.message})`)
+      return c || null
+    }
+  }
+
+  /** The account joined (`add`) or left (`remove`) a session on this machine. */
+  async hostedRoom (account, { add, remove }) {
+    const c = this.hosted.get(account)
+    if (c) {
+      if (add && !c.rooms.includes(add)) c.rooms.push(add)
+      if (remove) c.rooms = c.rooms.filter((r) => r !== remove)
+    }
+    try {
+      const r = await this.call('POST', `/v1/relay/hosted/${encodeURIComponent(account)}/rooms`, { ...(add ? { add } : {}), ...(remove ? { remove } : {}) })
+      if (c && Array.isArray(r.rooms)) c.rooms = r.rooms
+    } catch (err) { this.log(`cluster: could not note ${account}'s sessions (${err.message})`) }
+  }
+
+  async hostedWebhook (account, webhook) {
+    const c = this.hosted.get(account)
+    if (c) c.webhook = webhook || { off: true }
+    try { await this.call('PUT', `/v1/relay/hosted/${encodeURIComponent(account)}/webhook`, { webhook }) } catch (err) { this.log(`cluster: could not keep ${account}'s webhook (${err.message})`) }
+  }
+
+  /**
+   * Whether `account` owns session `room` or has been in it, from the API (cached): true/false,
+   * or undefined while it is being asked (then `then` runs with the answer).
+   */
+  sameOwner (room, account, then = () => {}) {
+    const k = `${room}\n${account}`
+    const c = this.owners.get(k)
+    if (c && this.now() - c.at < SAME_OWNER_TTL_MS) return c.same
+    if (!c || !c.asking) {
+      const entry = { same: c ? c.same : undefined, at: c ? c.at : 0, asking: true }
+      this.owners.set(k, entry)
+      this.call('POST', '/v1/relay/same-owner', { room, account })
+        .then((r) => { const same = r.same === true; const changed = same !== entry.same; Object.assign(entry, { same, at: this.now(), asking: false }); if (changed) then(same) })
+        .catch((err) => { entry.asking = false; this.log(`cluster: could not ask who is in ${room} (${err.message})`) })
+      if (this.owners.size > 10000) this.owners.delete(this.owners.keys().next().value)
+    }
+    return c ? c.same : undefined
+  }
+}
+
+// ---- a room's files, from one machine to another ----
+//
+// The stream is frames: a JSON line ({ path, size }) and that many bytes; the last line is
+// { end: true, files, bytes }. Paths are relative to the data folder and belong to the room.
+
+/** The paths of the room's files, relative to the data folder. */
+export function roomPaths (dataDir, room) {
+  const out = []
+  const walk = (rel) => {
+    const abs = path.join(dataDir, rel)
+    let st
+    try { st = fs.statSync(abs) } catch { return }
+    if (st.isDirectory()) { for (const f of fs.readdirSync(abs).sort()) walk(path.posix.join(rel, f)) } else if (st.isFile()) out.push(rel)
+  }
+  for (const rel of [`${room}.ydoc`, `${room}.json`, `${room}.chat.jsonl`, `branches/${room}`, `files/${room}`, `blobs/${room}`]) walk(rel)
+  return out
+}
+
+/** Whether `rel` is one of `room`'s own paths (nothing else is written on import). */
+export function roomOwns (room, rel) {
+  if (typeof rel !== 'string' || rel.includes('\\') || rel.split('/').some((p) => p === '..' || p === '.' || p === '')) return false
+  return rel === `${room}.ydoc` || rel === `${room}.json` || rel === `${room}.chat.jsonl` || [`branches/${room}/`, `files/${room}/`, `blobs/${room}/`].some((p) => rel.startsWith(p))
+}
+
+/** Writes room `room`'s files (and `extras`, JSON) to `res` as frames, a file at a time from disk. */
+export async function packRoom (dataDir, room, extras, res) {
+  let files = 0
+  let bytes = 0
+  const write = (chunk) => new Promise((resolve, reject) => {
+    const fail = (err) => reject(err)
+    res.once('error', fail)
+    const done = () => { res.off('error', fail); resolve() }
+    if (res.write(chunk)) done(); else res.once('drain', done)
+  })
+  const x = Buffer.from(JSON.stringify(extras || {}))
+  await write(`${JSON.stringify({ path: ':extras', size: x.length })}\n`)
+  await write(x)
+  for (const rel of roomPaths(dataDir, room)) {
+    const abs = path.join(dataDir, rel)
+    const size = fs.statSync(abs).size
+    await write(`${JSON.stringify({ path: rel, size })}\n`)
+    let sent = 0
+    for await (const chunk of fs.createReadStream(abs, { end: Math.max(0, size - 1), highWaterMark: 1024 * 1024 })) {
+      if (!size) break
+      await write(chunk)
+      sent += chunk.length
+    }
+    if (sent !== size) throw new Error(`${rel} changed while it was being copied`)
+    files++
+    bytes += size
+  }
+  res.end(`${JSON.stringify({ end: true, files, bytes })}\n`)
+  return { files, bytes }
+}
+
+/**
+ * Reads frames from `stream` into `stage` (a fresh folder), checking each path belongs to `room`,
+ * writing each file as its bytes come. Resolves { files, bytes, extras } once the end line matches
+ * what came; rejects otherwise.
+ */
+export async function unpackRoom (stream, room, stage) {
+  fs.rmSync(stage, { recursive: true, force: true })
+  fs.mkdirSync(stage, { recursive: true })
+  let head = Buffer.alloc(0) // the start of a frame's line, until its newline comes
+  let want = null // { path, size, left, fd?, extras? }
+  let files = 0
+  let bytes = 0
+  let extras = {}
+  let end = null
+  const finish = () => {
+    if (want.path === ':extras') extras = JSON.parse(Buffer.concat(want.parts).toString('utf8') || '{}')
+    else { fs.closeSync(want.fd); files++; bytes += want.size }
+    want = null
+  }
+  try {
+    for await (const chunk of stream) {
+      let buf = chunk
+      while (buf.length && !end) {
+        if (!want) {
+          const nl = buf.indexOf(10)
+          if (nl < 0) { head = Buffer.concat([head, buf]); if (head.length > 4096) throw new Error('a malformed frame'); buf = buf.subarray(buf.length); break }
+          const line = JSON.parse(Buffer.concat([head, buf.subarray(0, nl)]).toString('utf8'))
+          head = Buffer.alloc(0)
+          buf = buf.subarray(nl + 1)
+          if (line.end) { end = line; break }
+          if (!(Number.isInteger(line.size) && line.size >= 0)) throw new Error('a malformed frame')
+          if (line.path === ':extras') {
+            if (line.size > 16 * 1024 * 1024) throw new Error('a malformed frame')
+            want = { path: line.path, size: line.size, left: line.size, parts: [] }
+          } else {
+            if (!roomOwns(room, line.path)) throw new Error(`a file that isn't the room's: ${String(line.path).slice(0, 100)}`)
+            const abs = path.join(stage, line.path)
+            fs.mkdirSync(path.dirname(abs), { recursive: true })
+            want = { path: line.path, size: line.size, left: line.size, fd: fs.openSync(abs, 'w') }
+          }
+          if (!want.left) finish()
+          continue
+        }
+        const part = buf.subarray(0, want.left)
+        buf = buf.subarray(part.length)
+        if (want.parts) want.parts.push(part); else fs.writeSync(want.fd, part)
+        want.left -= part.length
+        if (!want.left) finish()
+      }
+      if (end) break
+    }
+  } finally {
+    if (want && want.fd !== undefined) fs.closeSync(want.fd)
+  }
+  if (!end || want || end.files !== files || end.bytes !== bytes) throw new Error(`the copy was cut short (${files} of ${end ? end.files : '?'} files)`)
+  return { files, bytes, extras }
+}
+
+/** Moves a staged room into the data folder, replacing whatever copy of it was there. */
+export function placeStaged (dataDir, room, stage) {
+  for (const rel of [`${room}.ydoc`, `${room}.json`, `${room}.chat.jsonl`, `branches/${room}`, `files/${room}`, `blobs/${room}`]) {
+    const from = path.join(stage, rel)
+    const to = path.join(dataDir, rel)
+    fs.rmSync(to, { recursive: true, force: true })
+    if (!fs.existsSync(from)) continue
+    fs.mkdirSync(path.dirname(to), { recursive: true })
+    fs.renameSync(from, to)
+  }
+  fs.rmSync(stage, { recursive: true, force: true })
+}
+
+/** Deletes this machine's copy of a room that lives elsewhere now (never its stored large files elsewhere). */
+export function removeLocalRoom (dataDir, room) {
+  for (const rel of [`${room}.ydoc`, `${room}.json`, `${room}.chat.jsonl`, `branches/${room}`, `files/${room}`, `blobs/${room}`]) fs.rmSync(path.join(dataDir, rel), { recursive: true, force: true })
+}
+
+/** Moves a room this machine should not hold aside, to `displaced/` (never served, never deleted). */
+export function displaceRoom (dataDir, room, at = Date.now()) {
+  const dest = path.join(dataDir, 'displaced', `${room}-${at}`)
+  for (const rel of [`${room}.ydoc`, `${room}.json`, `${room}.chat.jsonl`, `branches/${room}`, `files/${room}`]) {
+    const from = path.join(dataDir, rel)
+    if (!fs.existsSync(from)) continue
+    fs.mkdirSync(path.dirname(path.join(dest, rel)), { recursive: true })
+    fs.renameSync(from, path.join(dest, rel))
+  }
+  return dest
+}