heyquilt 0.3.90 → 0.3.107

package/src/api/releases.js

@@ -0,0 +1,87 @@
+// Quilt's own downloads and update check, now that the repository with the releases is private.
+// The API reads the latest GitHub release with Quilt's GitHub App (read-only, that repository
+// only) and answers on its own address, so nobody's download depends on the repository being
+// public:
+//
+//   GET /v1/releases/latest    the release, shaped like GitHub's (tag_name, published_at, body…),
+//                              with each file's link on this API; the app's update check reads it
+//   GET /v1/download/<file>    a redirect to GitHub's own short-lived link for that file of the
+//                              latest release: the bytes never pass through here
+//   GET /v1/download/v1.2.3/<file>   the same, from that version's release
+import { githubApp } from './github-app.js'
+
+export const RELEASES_REPO = { owner: 'DanielCarmichaelGit', name: 'heyquilt' }
+const FILE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/
+const TAG = /^v\d+\.\d+\.\d+$/
+const RELEASE_CACHE_MS = 60 * 1000
+const API = 'https://api.github.com'
+
+export class ReleaseError extends Error {
+  constructor (status, message) { super(message); this.status = status }
+}
+
+/** Reads the latest release of `repo` as the GitHub App. `fetch` and `now` are for tests. */
+export function releaseSource ({ github, fetch = globalThis.fetch, now = Date.now, repo = RELEASES_REPO, apiUrl = 'https://api.heyquilt.com' }) {
+  if (!github) return null
+  const app = githubApp(github, { fetch, now })
+  let token = null // { token, expiresAt }
+  let release = null // { at, value }
+  const tagged = new Map() // tag -> release (a published release's files don't change)
+  const auth = async () => {
+    if (token && token.expiresAt - now() > 5 * 60 * 1000) return token.token
+    const installation = await app.installation(repo.owner, repo.name)
+    if (!installation) throw new ReleaseError(503, `Quilt's GitHub App is not installed on ${repo.owner}/${repo.name}`)
+    token = await app.repoToken(installation, repo.name, { access: 'read' })
+    return token.token
+  }
+  const get = async (path, { accept = 'application/vnd.github+json', redirect = 'follow' } = {}) => fetch(`${API}${path}`, {
+    redirect,
+    headers: { 'user-agent': 'quilt-api', accept, 'x-github-api-version': '2022-11-28', authorization: `Bearer ${await auth()}` }
+  })
+  return {
+    /** The latest release, as GitHub has it ({ tag_name, assets, … }); asked at most once a minute. */
+    async latest () {
+      if (release && now() - release.at < RELEASE_CACHE_MS) return release.value
+      const res = await get(`/repos/${repo.owner}/${repo.name}/releases/latest`)
+      if (!res.ok) {
+        if (release) return release.value // GitHub is having a moment: the last answer stands
+        throw new ReleaseError(res.status === 404 ? 404 : 502, res.status === 404 ? 'there is no release yet' : `GitHub answered ${res.status}`)
+      }
+      release = { at: now(), value: await res.json() }
+      return release.value
+    },
+    /** The latest release as the app's update check reads it, every file linked on this API. */
+    async latestPublic () {
+      const r = await this.latest()
+      return {
+        tag_name: r.tag_name,
+        name: r.name,
+        html_url: 'https://heyquilt.com/download',
+        published_at: r.published_at,
+        body: r.body || '',
+        assets: (r.assets || []).map((a) => ({ name: a.name, size: a.size, browser_download_url: `${apiUrl}/v1/download/${encodeURIComponent(a.name)}` }))
+      }
+    },
+    /** The release tagged `tag` (v1.2.3). */
+    async byTag (tag) {
+      if (!TAG.test(String(tag))) throw new ReleaseError(404, 'no such version')
+      if (tagged.has(tag)) return tagged.get(tag)
+      const res = await get(`/repos/${repo.owner}/${repo.name}/releases/tags/${tag}`)
+      if (!res.ok) throw new ReleaseError(res.status === 404 ? 404 : 502, res.status === 404 ? `there is no release ${tag}` : `GitHub answered ${res.status}`)
+      const value = await res.json()
+      if (tagged.size > 200) tagged.clear()
+      tagged.set(tag, value)
+      return value
+    },
+    /** GitHub's short-lived link for `file` in the latest release, or in release `tag`. */
+    async downloadUrl (file, tag = '') {
+      if (!FILE.test(String(file))) throw new ReleaseError(404, 'no such file')
+      const asset = ((tag ? await this.byTag(tag) : await this.latest()).assets || []).find((a) => a.name === file)
+      if (!asset) throw new ReleaseError(404, `${tag ? `release ${tag}` : 'the latest release'} has no ${file}`)
+      const res = await get(`/repos/${repo.owner}/${repo.name}/releases/assets/${asset.id}`, { accept: 'application/octet-stream', redirect: 'manual' })
+      const to = res.headers.get('location')
+      if (res.status >= 300 && res.status < 400 && to) return to
+      throw new ReleaseError(502, `GitHub answered ${res.status} for ${file}`)
+    }
+  }
+}

package/src/api/routes/agents.js

@@ -3,18 +3,22 @@
 import { HttpError, needId, cleanName } from '../http.js'
 import { parsePublicKey, verifyAgentResume } from '../../identity.js'
 import { keyStatus } from '../agent-auth.js'
-import { encodeAgentEnvKey } from '../../ui/agent-env-key.js'
+import { encodeAgentEnvKey, decodeAgentEnvKey } from '../../ui/agent-env-key.js'
+import { usesByDay, lastDays, keyState, linkSealer } from '../sandbox-keys.js'
 
 // Live app keys an agent may hold at once.
 export const MAX_APP_KEYS = 10
 const keyName = (v) => cleanName(typeof v === 'string' ? v : '', 40, 'name the key (up to 40 characters), e.g. Pipedream')
-const keyView = (k) => ({ id: k.id, name: k.name, createdAt: k.createdAt, lastUsedAt: k.lastUsedAt })
+const keyView = (k) => ({ id: k.id, name: k.name, createdAt: k.createdAt, lastUsedAt: k.lastUsedAt, ...(k.kind === 'sandbox' ? { sandbox: true, hint: k.hint } : {}) })
+const sessionName = (v) => (typeof v === 'string' ? v.trim().slice(0, 80) : '')
 
 // Never the key itself, just whether it has one. With a key it joins sessions from a computer
 // running Quilt; without one it is hosted: it joins through the API's /mcp. Either way it can join.
 const profileOf = (a) => ({ id: a.id, name: a.name, provider: a.provider, type: a.type, description: a.description, canJoinSessions: true, hosted: !a.publicKey })
 
-export function agentRoutes ({ store, user, person, now, limit, limitTokens, limitStarts, spendResume, resumeWindowMs, agentAuth, apiUrl }) {
+export function agentRoutes ({ store, user, person, now, limit, limitTokens, limitStarts, spendResume, resumeWindowMs, agentAuth, apiUrl, passKey = '' }) {
+  // Sandbox keys keep their session's invite link sealed (sandbox-keys.js), for Rotate.
+  const links = linkSealer(passKey)
   // A personal agent of this person's that isn't revoked; anyone else's gets the same 404 as a missing one.
   async function mine (req, id, who = user) {
     const u = await who(req)
@@ -22,19 +26,56 @@ export function agentRoutes ({ store, user, person, now, limit, limitTokens, lim
     if (!agent || agent.ownerUserId !== u.userId || agent.revokedAt) throw new HttpError(404, 'no such agent')
     return agent
   }
-  async function newAppKey (agent, name) {
-    if ((await store.listAgentAppKeys(agent.id)).length >= MAX_APP_KEYS) throw new HttpError(409, `an agent can have ${MAX_APP_KEYS} app keys; revoke one first`)
-    return agentAuth.mintAppKey(agent.id, name)
+  // `rotating`: a key that replaces one, so the agent holds no more keys than before.
+  async function newAppKey (agent, name, sandbox = null, rotating = false) {
+    if (!rotating && (await store.listAgentAppKeys(agent.id)).length >= MAX_APP_KEYS) throw new HttpError(409, `an agent can have ${MAX_APP_KEYS} app keys; revoke one first`)
+    return agentAuth.mintAppKey(agent.id, name, sandbox)
   }
   const connect = (agent, key) => ({ agent: profileOf(agent), key, mcp: `${apiUrl}/mcp` })
-  // A sandbox's QUILT_AGENT_KEY (ui/agent-env-key.js): a new app key with the API and the session it works in.
-  async function envKeyFor (agent, session, name) {
-    let envKey
-    const check = { api: apiUrl, key: 'qk_check', session, name: agent.name }
-    try { encodeAgentEnvKey(check) } catch (err) { throw new HttpError(400, err.message) }
-    const key = await newAppKey(agent, keyName(name || 'Sandbox'))
-    try { envKey = encodeAgentEnvKey({ ...check, key: key.key }) } catch (err) { throw new HttpError(400, err.message) }
-    return { envKey, key: keyView(key) }
+  /**
+   * A sandbox's QUILT_AGENT_KEY (ui/agent-env-key.js): a new app key with the API and the session it
+   * works in, kept as a sandbox key (its session, sync slot and hint). `slot` carries a rotated key's on.
+   */
+  async function envKeyFor (agent, { session, name, label = '', slot, rotating = false }) {
+    let made
+    const check = { api: apiUrl, key: 'qk_check', session, slot, name: agent.name }
+    try { made = decodeAgentEnvKey(encodeAgentEnvKey(check)) } catch (err) { throw new HttpError(400, err.message) }
+    const room = made.room
+    const known = await store.sessionByRoom(room).catch(() => null)
+    // A session keeps the name its other keys show (the app's label is only its folder's name).
+    const prior = agent.ownerUserId ? (await personSandboxKeys(agent.ownerUserId)).find((k) => k.room === room && k.sessionName)?.sessionName : null
+    const key = await newAppKey(agent, keyName(name || 'Sandbox'), { room, sessionLink: links.seal(made.session), sessionName: known?.name || prior || sessionName(label) || null, slot: made.slot }, rotating)
+    const envKey = encodeAgentEnvKey({ ...check, session: made.session, slot: made.slot, key: key.key })
+    return { envKey, key: (await sandboxViews([await store.agentAppKeyById(key.id)]))[0] }
+  }
+  /** Sandbox keys as the Keys table shows them: never the key, the link or the slot. */
+  async function sandboxViews (rows) {
+    const at = now()
+    const uses = await store.appKeyUses(rows.map((k) => k.id), lastDays(at)[0])
+    const names = new Map()
+    for (const room of new Set(rows.map((k) => k.room))) names.set(room, (await store.sessionByRoom(room).catch(() => null))?.name || '')
+    return rows.map((k) => ({
+      id: k.id,
+      name: k.name,
+      hint: k.hint,
+      room: k.room,
+      session: names.get(k.room) || k.sessionName || k.room,
+      createdAt: k.createdAt,
+      lastUsedAt: k.lastUsedAt,
+      revokedAt: k.revokedAt,
+      uses: usesByDay(uses.filter((u) => u.keyId === k.id), at),
+      state: keyState(k, at)
+    }))
+  }
+  /** The person's sandbox keys across their agents: the sessions they can make one for by room. */
+  async function personSandboxKeys (userId) {
+    return store.listSandboxKeys((await store.listPersonalAgents(userId)).map((a) => a.id))
+  }
+  /** A sandbox key of this agent's, or the same 404 as a missing one. */
+  async function sandboxKey (agent, keyId) {
+    const k = await store.agentAppKeyById(needId(keyId, 'key'))
+    if (!k || k.agentId !== agent.id || k.kind !== 'sandbox') throw new HttpError(404, 'no such key')
+    return k
   }
 
   return [
@@ -116,16 +157,70 @@ export function agentRoutes ({ store, user, person, now, limit, limitTokens, lim
       if (!keyRow) throw new HttpError(403, 'sign in with the agent\'s access key (an app key can\'t make keys)')
       if (!agent.ownerUserId) throw new HttpError(403, 'only a personal agent makes its own keys; an org agent\'s are made by its org')
       limit(req)
-      if (body.session != null && body.session !== '') return envKeyFor(agent, String(body.session), body.name)
+      if (body.session != null && body.session !== '') return envKeyFor(agent, { session: String(body.session), name: body.name })
       return newAppKey(agent, keyName(body.name || 'Sandbox'))
     }],
 
     // A person making a sandbox's QUILT_AGENT_KEY for their agent (the website, the app): a new app key
     // with the session it works in. Shown once; revoked like any of the agent's app keys.
+    // `room` instead of `session`: a session one of this person's sandbox keys already works in
+    // (the website's session picker, which never has the link). `sessionName` names it in the table.
     ['POST', /^\/v1\/agents\/([^/]+)\/env-key$/, async (req, body, [id]) => {
       const agent = await mine(req, id, person)
       limit(req)
-      return envKeyFor(agent, String(body.session || ''), body.name)
+      let session = String(body.session || '')
+      if (!session && body.room) {
+        const u = await person(req)
+        const known = (await personSandboxKeys(u.userId)).filter((k) => k.room === String(body.room))
+        session = known.map((k) => links.open(k.sessionLink)).find(Boolean) || ''
+        if (!session) throw new HttpError(400, 'paste the session\'s invite link')
+      }
+      return envKeyFor(agent, { session, name: body.name, label: body.sessionName })
+    }],
+
+    // The Keys table on an agent: its sandbox keys (revoked ones until removed), each with its
+    // session, last use, requests per day over the last seven days and state; and the sessions
+    // a new key can be made for without pasting a link.
+    ['GET', /^\/v1\/agents\/([^/]+)\/sandbox-keys$/, async (req, body, [id]) => {
+      const agent = await mine(req, id, person)
+      const u = await person(req)
+      const all = await personSandboxKeys(u.userId)
+      const sessions = new Map()
+      for (const k of all) if (!sessions.has(k.room) && links.open(k.sessionLink)) sessions.set(k.room, k)
+      const named = await sandboxViews([...sessions.values()])
+      return {
+        keys: await sandboxViews(all.filter((k) => k.agentId === agent.id)),
+        sessions: named.map((k) => ({ room: k.room, name: k.session }))
+      }
+    }],
+
+    // Rotate: a new key for the same agent, session, name and sync slot (shown once), and the old one revoked.
+    ['POST', /^\/v1\/agents\/([^/]+)\/sandbox-keys\/([^/]+)\/rotate$/, async (req, body, [id, keyId]) => {
+      const agent = await mine(req, id, person)
+      const old = await sandboxKey(agent, keyId)
+      if (old.revokedAt) throw new HttpError(404, 'no such key')
+      limit(req)
+      const session = links.open(old.sessionLink)
+      if (!session) throw new HttpError(409, 'make a new key for this session instead (paste its invite link)')
+      const made = await envKeyFor(agent, { session, name: old.name, label: old.sessionName || '', slot: old.slot, rotating: true })
+      await store.revokeAgentAppKey(agent.id, old.id)
+      return { ...made, replaced: old.id }
+    }],
+
+    ['POST', /^\/v1\/agents\/([^/]+)\/sandbox-keys\/([^/]+)\/revoke$/, async (req, body, [id, keyId]) => {
+      const agent = await mine(req, id, person)
+      const k = await sandboxKey(agent, keyId)
+      if (!await store.revokeAgentAppKey(agent.id, k.id)) throw new HttpError(404, 'no such key')
+      return { ok: true }
+    }],
+
+    // Remove: a revoked key's row leaves the table (its use counts go with it).
+    ['DELETE', /^\/v1\/agents\/([^/]+)\/sandbox-keys\/([^/]+)$/, async (req, body, [id, keyId]) => {
+      const agent = await mine(req, id, person)
+      const k = await sandboxKey(agent, keyId)
+      if (!k.revokedAt) throw new HttpError(409, 'revoke the key first')
+      await store.removeAgentAppKey(agent.id, k.id)
+      return { ok: true }
     }],
 
     ['GET', /^\/v1\/agents\/([^/]+)\/keys$/, async (req, body, [id]) => {

package/src/api/routes/ai-vm.js

@@ -8,7 +8,7 @@
 // or the app collects them: none is ever stored. Quilt never sees a Claude, OpenAI or Cursor token.
 import crypto from 'node:crypto'
 import { HttpError, UUID } from '../http.js'
-import { newToken, hashToken } from '../tokens.js'
+import { newToken, hashToken, sameHash } from '../tokens.js'
 import { AGENT_INVITE_TTL_MS } from './agent-invites.js'
 import { parseInvite, buildInvite } from '../../ui/invite.js'
 import { isHostedRelay, HOSTED_RELAY } from '../../settings.js'
@@ -27,6 +27,11 @@ const CONTROL = /[\u0000-\u001f\u007f]/g // eslint-disable-line no-control-regex
 const STEPS = ['starting', 'link', 'code', 'waiting', 'done', 'failed']
 const CODE = /^[A-Za-z0-9-]{1,40}$/
 const ROOM = /^[A-Za-z0-9_-]{1,64}$/
+// How often the spare machines are checked (made again, or replaced for a new image).
+export const SPARES_EVERY_MS = 10 * 60 * 1000
+// A spare not ready by then is made again; a claimed spare's join link is kept this long.
+const SPARE_STUCK_MS = 15 * 60 * 1000
+const SPARE_CLAIM_MS = 15 * 60 * 1000
 const digest = (s) => crypto.createHash('sha256').update(String(s)).digest()
 
 /** What the person's app shows of a sign-in step: a link, a device code, whether to paste a code back. */
@@ -63,15 +68,15 @@ export async function cloudAiOf (store, agent) {
   return store.updateAiVm(theirs.userId, { agentId: agent.id })
 }
 
-export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log = () => {}, limit, relaySecret = '', relayUrl = HOSTED_RELAY, fly = null, waitMs = WAIT_MS }) {
-  // user id -> { version, signIn, sessions, waiters, forVm: { follow, leave, signIn, code, waiters } }
+export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log = () => {}, limit, relaySecret = '', relayUrl = HOSTED_RELAY, fly = null, waitMs = WAIT_MS, sparesEveryMs = SPARES_EVERY_MS, spareSuspendMs = 1000 }) {
+  // user id -> { version, signIn, waiters, forVm: { follow, leave, signIn, code, config, waiters } }
   const live = new Map()
   const ownRelay = String(relayUrl).replace(/\/+$/, '').replace(/^http(s?):\/\//, 'ws$1://')
   const relayWokeAt = new Map()
 
   const liveOf = (userId) => {
     let l = live.get(userId)
-    if (!l) { l = { version: 1, signIn: null, sessions: [], waiters: new Set(), forVm: { follow: [], leave: [], signIn: false, code: null, waiters: new Set() } }; live.set(userId, l) }
+    if (!l) { l = { version: 1, signIn: null, waiters: new Set(), forVm: { follow: [], leave: [], signIn: false, code: null, config: false, waiters: new Set() } }; live.set(userId, l) }
     return l
   }
   const wakeWaiters = (set) => { for (const w of [...set]) w(); set.clear() }
@@ -85,6 +90,105 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
     set.add(done)
   })
 
+  // Spare machines (fly-machines.js), by id: { id, volumeId, region, hash, state, ready }. A
+  // spare reads the AI tools into memory, says it is ready and is suspended; connecting resumes it.
+  const spares = new Map()
+  // Spares handed to someone, by id: { link, hash, at }. The machine asks for its join link with
+  // its spare key; the link is kept a while, so a machine that restarts first can ask again.
+  const claimed = new Map()
+  let filling = null
+
+  /** Makes spares up to `fly.spares`, and replaces those of an older image or that never got ready. One run at a time. */
+  function fillSpares () {
+    if (!fly || !fly.spares || !fly.listSpares) return Promise.resolve()
+    if (filling) return filling
+    filling = (async () => {
+      for (const [id, c] of claimed) if (now() - c.at > SPARE_CLAIM_MS) claimed.delete(id)
+      const there = await fly.listSpares()
+      const seen = new Set()
+      for (const m of there) {
+        if (claimed.has(m.id)) continue
+        const had = spares.get(m.id)
+        const ready = m.state === 'suspended' || !!(had && had.ready)
+        const stuck = !ready && m.createdAt && Date.now() - m.createdAt > SPARE_STUCK_MS
+        if (m.image !== fly.image || !m.volumeId || !m.hash || stuck) {
+          spares.delete(m.id)
+          await fly.destroyMachine(m.id).catch((err) => log(`cloud AI spare ${m.id}: could not delete it: ${err.message}`))
+          if (m.volumeId) await fly.destroyVolume(m.volumeId).catch(() => {})
+          continue
+        }
+        seen.add(m.id)
+        spares.set(m.id, { id: m.id, volumeId: m.volumeId, region: m.region, hash: m.hash, state: m.state, ready })
+      }
+      for (const id of [...spares.keys()]) if (!seen.has(id)) spares.delete(id)
+      while (spares.size < fly.spares) {
+        const short = crypto.randomBytes(10).toString('hex')
+        const key = newToken('qs_')
+        const volume = await fly.createVolume({ name: `ai_s_${short}` })
+        try {
+          const m = await fly.createMachine({ name: `ai-s-${short}`, volumeId: volume.id, env: { QUILT_API: apiUrl, QUILT_AI_VM_SPARE: key, HOME: '/data/home' }, spareHash: hashToken(key) })
+          spares.set(m.id, { id: m.id, volumeId: volume.id, region: m.region || volume.region || fly.region, hash: hashToken(key), state: 'started', ready: false })
+          log(`cloud AI spare ${m.id}: made`)
+        } catch (err) {
+          await fly.destroyVolume(volume.id).catch(() => {})
+          throw err
+        }
+      }
+    })().catch((err) => log(`cloud AI spares: ${err.message}`)).finally(() => { filling = null })
+    return filling
+  }
+  let sparesTimer = null
+  if (fly && fly.spares) {
+    sparesTimer = setInterval(() => { fillSpares() }, sparesEveryMs)
+    sparesTimer.unref?.()
+    setImmediate(() => { fillSpares() })
+  }
+
+  /** The spare calling, by its key: { id, spare } before it is claimed, { id, claim } after. */
+  function spareFrom (req) {
+    const key = bearer(req)
+    if (!key || !key.startsWith('qs_')) throw new HttpError(401, 'a spare key is needed')
+    const hash = hashToken(key)
+    for (const [id, c] of claimed) if (sameHash(c.hash, hash)) return { id, claim: c }
+    for (const sp of spares.values()) if (sameHash(sp.hash, hash)) return { id: sp.id, spare: sp }
+    throw new HttpError(401, 'no such spare')
+  }
+
+  /**
+   * The person's machine: a spare when there is one (a ready one first: it is running, tools in
+   * memory, in a second or two), else a new volume and machine. A spare Fly refuses is deleted.
+   */
+  async function machineFor (userId, env) {
+    while (spares.size) {
+      const spare = [...spares.values()].sort((a, b) => Number(b.ready) - Number(a.ready))[0]
+      spares.delete(spare.id)
+      claimed.set(spare.id, { link: env.QUILT_AI_VM_JOIN, hash: spare.hash, at: now() })
+      try {
+        // One being suspended right now is woken when that is done (the ready route).
+        await fly.claimSpare(spare.id, { state: spare.suspending ? 'started' : spare.state })
+        fillSpares()
+        return { machineId: spare.id, volumeId: spare.volumeId, region: spare.region }
+      } catch (err) {
+        log(`cloud AI spare ${spare.id}: could not use it: ${err.message}`)
+        claimed.delete(spare.id)
+        // Fly busy or briefly down: the spare is fine, and stays for the next person. Else it goes.
+        if (err.status === 429 || err.status >= 500) { spares.set(spare.id, spare); break }
+        await fly.destroyMachine(spare.id).catch(() => {})
+        await fly.destroyVolume(spare.volumeId).catch(() => {})
+      }
+    }
+    fillSpares()
+    const short = crypto.createHash('sha256').update(userId).digest('hex').slice(0, 20)
+    const volume = await fly.createVolume({ name: `ai_${short}` })
+    try {
+      const m = await fly.createMachine({ name: `ai-${short}`, volumeId: volume.id, env })
+      return { machineId: m.id, volumeId: volume.id, region: m.region || volume.region || fly.region }
+    } catch (err) {
+      await fly.destroyVolume(volume.id).catch(() => {})
+      throw err
+    }
+  }
+
   const needFly = () => { if (!fly) throw new HttpError(503, "Cloud AI isn't set up on this server.") }
   async function mine (req) {
     const u = await person(req)
@@ -106,7 +210,8 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
       error: vm.error,
       signedIn: vm.signedIn,
       signIn: l.signIn,
-      sessions: l.sessions,
+      // Kept with the record: shown while the machine sleeps, and after the API restarts.
+      sessions: Array.isArray(vm.sessions) ? vm.sessions : [],
       usageToday: vm.usageToday || {},
       region: vm.region,
       lastReportAt: vm.lastReportAt,
@@ -156,7 +261,8 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
     return { agent, vm }
   }
 
-  const pendingFor = (f) => f.follow.length || f.leave.length || f.signIn || f.code !== null
+  // `config`: the settings changed since the machine last asked (a new tool signs in at once).
+  const pendingFor = (f) => f.follow.length || f.leave.length || f.signIn || f.code !== null || f.config
 
   function cleanSettings (b, vm) {
     const out = {}
@@ -205,19 +311,11 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
       const invite = await store.createAgentInvite({ tokenHash: hashToken(token), ownerUserId: u.userId, createdBy: u.userId, expiresAt: now() + AGENT_INVITE_TTL_MS })
       let vm = await store.createAiVm({ userId: u.userId, tool: body.tool, region: fly.region })
       vm = await store.updateAiVm(u.userId, { inviteId: invite.id })
-      const short = crypto.createHash('sha256').update(u.userId).digest('hex').slice(0, 20)
-      let volume = null
       try {
-        volume = await fly.createVolume({ name: `ai_${short}` })
-        const m = await fly.createMachine({
-          name: `ai-${short}`,
-          volumeId: volume.id,
-          env: { QUILT_API: apiUrl, QUILT_AI_VM_JOIN: `${apiUrl}/v1/join/${token}`, HOME: '/data/home' }
-        })
-        vm = await store.updateAiVm(u.userId, { machineId: m.id, volumeId: volume.id, region: m.region || volume.region || fly.region, state: 'starting' })
+        const m = await machineFor(u.userId, { QUILT_API: apiUrl, QUILT_AI_VM_JOIN: `${apiUrl}/v1/join/${token}`, HOME: '/data/home' })
+        vm = await store.updateAiVm(u.userId, { ...m, state: 'starting' })
       } catch (err) {
         log(`cloud AI ${u.userId}: could not create it: ${err.message}`)
-        if (volume) await fly.destroyVolume(volume.id).catch(() => {})
         await store.cancelAgentInvite(invite.id).catch(() => {})
         await store.deleteAiVm(u.userId)
         throw new HttpError(502, `Couldn't start your cloud AI. ${err.message}`)
@@ -232,6 +330,9 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
       const patch = cleanSettings(body, vm)
       if (!Object.keys(patch).length) return view(vm)
       let up = await store.updateAiVm(u.userId, patch)
+      liveOf(u.userId).forVm.config = true
+      // A new tool's sign-in starts over: the old tool's link and code are not shown for it.
+      if (patch.tool) liveOf(u.userId).signIn = { step: 'starting', url: '', code: '', needsCode: false, error: '' }
       bump(u.userId)
       poke(u.userId)
       up = await wake(up, 'settings')
@@ -310,7 +411,7 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
         signIn: f.signIn,
         code: f.code
       }
-      Object.assign(f, { follow: [], leave: [], signIn: false, code: null })
+      Object.assign(f, { follow: [], leave: [], signIn: false, code: null, config: false })
       return out
     }],
 
@@ -321,7 +422,7 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
       if (typeof body.signedIn === 'boolean') patch.signedIn = body.signedIn
       if (body.usageToday !== undefined) patch.usageToday = cleanUsage(body.usageToday)
       if (body.signIn !== undefined) l.signIn = cleanSignIn(body.signIn)
-      if (body.sessions !== undefined) l.sessions = cleanSessions(body.sessions)
+      if (body.sessions !== undefined) patch.sessions = cleanSessions(body.sessions)
       await store.updateAiVm(vm.userId, patch)
       bump(vm.userId)
       return { ok: true }
@@ -342,6 +443,30 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
       return { ok: true }
     }],
 
+    // A spare machine has read the tools into memory: it is suspended (after it has had its
+    // answer). It asks again while awake; once someone has taken it, the answer is its join link.
+    ['POST', /^\/v1\/ai-vm\/spare\/ready$/, async (req) => {
+      const s = spareFrom(req)
+      if (s.claim) return { join: s.claim.link }
+      const sp = s.spare
+      if (!sp.ready) log(`cloud AI spare ${sp.id}: ready`)
+      sp.ready = true
+      if (fly && !sp.suspending) {
+        sp.suspending = new Promise((resolve) => setTimeout(resolve, spareSuspendMs)).then(async () => {
+          if (!claimed.has(sp.id)) { await fly.suspend(sp.id); sp.state = 'suspended' }
+          // Taken while it was being suspended: wake it for its person.
+          if (claimed.has(sp.id)) await fly.start(sp.id)
+        }).catch((err) => log(`cloud AI spare ${sp.id}: could not suspend it: ${err.message}`)).finally(() => { sp.suspending = null })
+      }
+      return { join: null }
+    }],
+
+    // A spare still reading its tools, asking whether someone has taken it: its join link if so.
+    ['GET', /^\/v1\/ai-vm\/spare\/claim$/, async (req) => {
+      const s = spareFrom(req)
+      return { join: s.claim ? s.claim.link : null }
+    }],
+
     // The relay: a mention, a message or a task waits for a cloud AI that isn't connected.
     ['POST', /^\/v1\/relay\/ai-vm\/wake$/, async (req, body) => {
       if (!relaySecret) throw new HttpError(503, 'the relay is not set up on this server')
@@ -365,5 +490,7 @@ export function aiVmRoutes ({ store, person, agentAuth, bearer, now, apiUrl, log
     if (vm) await destroy(vm)
   }
 
-  return { routes, destroyFor }
+  const close = () => { if (sparesTimer) clearInterval(sparesTimer) }
+
+  return { routes, destroyFor, fillSpares, close }
 }

package/src/api/routes/orgs.js

@@ -14,7 +14,7 @@ const sortRoles = (roles) => [...roles].sort((a, b) => (ROLE_ORDER[a.builtin] ??
 const orgView = (o) => ({ id: o.id, name: o.name, slug: o.slug, domain: o.domain, domainRequests: o.domainRequests, createdAt: o.createdAt })
 const roleView = (r) => ({ id: r.id, name: r.name, builtin: r.builtin, grants: normalizeGrants(r.grants), createdAt: r.createdAt })
 
-export function orgRoutes ({ store, user, person }) {
+export function orgRoutes ({ store, user, person, log = () => {} }) {
   const orgFor = async (req, slug) => { const u = await user(req); return { u, ...(await orgAccess(store, u.userId, slug)) } }
 
   // Two orgs made at once can pick the same slug; the unique index decides and we try the next.
@@ -101,6 +101,8 @@ export function orgRoutes ({ store, user, person }) {
     ['DELETE', /^\/v1\/orgs\/([^/]+)$/, async (req, body, [slug]) => {
       const a = await orgFor(req, slug)
       a.needOwner()
+      // Its team secrets go with it; never in the way of the delete.
+      await store.deleteScopeSecrets('org', a.org.id).catch((err) => log(`team secrets: ${err.message}`))
       await store.deleteOrg(a.org.id)
       return { ok: true }
     }],