heyquilt 0.3.90 → 0.3.107

package/src/server.js

@@ -60,12 +60,12 @@ import { leaseKeeper, LEASE_HEARTBEAT_MS } from './relay-lease.js'
 import { bringInBranch, BACKSTOP_MS, ACTIVE_BACKSTOP_MS, syncStatus, markFolderBrought } from './relay-sync.js'
 import { noteCiResult } from './relay-ci.js'
 import { outboundFor, replayBundle, liveSyncOn, adoptBranch, receiveAgentPush } from './relay-outbound.js'
-import { ensureView, syncViewRefs, serveGit, parseUpdates, viewRepoOf, viewDir } from './relay-githttp.js'
+import { ensureView, syncViewRefs, serveGit, parseUpdates, viewRepoOf, viewDir, gitHttpBackendWorks } from './relay-githttp.js'
 import { relayFileKey } from './relay-filekey.js'
-import { RoomMeter, roomDiskBytes } from './relay-metrics.js'
+import { RoomMeter, roomDiskBytes, MachineMeter, machineLoad } from './relay-metrics.js'
 import { clusterConfig, clusterProblems, Cluster, NOT_ROOMS, roomOfPath, replaySource, canReplay, displaceRoom, packRoom, unpackRoom, placeStaged, removeLocalRoom, MOVING, ELSEWHERE } from './relay-cluster.js'
 import { joiningRoom, sessionArg } from './mcp-room.js'
-import { clashAction, heldForClash, reviewClashes, noteMergeChecks, openOfflineClash } from './relay-clash.js'
+import { clashAction, noteMergeChecks } from './relay-clash.js'
 import { AGENT_COMMITS } from './github-commit.js'
 import { UPSTREAM_CHECK_MS, SYNC_EVERY_MS, SYNC_EVERY_GIT_MS, upstreamsOf, noteFolders, relayEntry, relayLooks, dueBranches, checkBranch, folderOn, connectedOn, hostedOn, validToken, folderAble, catcherAble, noteHolds, connectedAs, describeRelayCheck, loadBranch, parseRepo, configureGithubBudget, CLASH_IDLE_MS, noteConnection } from './relay-upstream.js'
 
@@ -142,6 +142,9 @@ export function relayConfig (opts = {}) {
     // GitHub as a node (relay-sync.js): commits come in through the relay's git mirror, on by default; QUILT_GIT_SYNC=0 keeps the REST path.
     gitSync: opts.gitSync ?? env.QUILT_GIT_SYNC !== '0',
     gitRemote: typeof opts.gitRemote === 'function' ? opts.gitRemote : null, // tests: a repository slug -> where its git is
+    // Agents' folders use git through the relay (relay-githttp.js); off when this machine's git has no http-backend.
+    gitHttp: opts.gitHttp !== false,
+    gitHttpCommand: typeof opts.gitHttpCommand === 'string' ? opts.gitHttpCommand : 'git', // tests: a git without http-backend
     syncAuto: opts.syncAuto === true, // the relay's own commit timer: off; commits happen when someone asks for one (quilt_git_commit, a git push)
     crashAt: typeof opts.crashAt === 'string' ? opts.crashAt : null, // tests: where to inject a crash (relay-sync.js crashPoint)
     leaseTtlMs: num(opts.leaseTtlMs, undefined),
@@ -719,8 +722,7 @@ class Room {
   mayBox (ws, rel) {
     const a = this.access.get(ws)
     rel = String(rel ?? '').trim().replace(/^\.\//, '')
-    if (a && !this.mayWrite(a, rel)) return false
-    return !(a && a.kind === 'agent' && this.cfg.gitSync && ws.branch && heldForClash(this, this.resolveKey(ws.branch), rel))
+    return !(a && !this.mayWrite(a, rel))
   }
 
   /**
@@ -778,8 +780,6 @@ class Room {
       }
     }
     for (const [rel, types] of touched) {
-      // A file held for a clash with GitHub (relay-clash.js): agents propose the merge with quilt_clash; only people edit it in place.
-      if (d.key && a && a.kind === 'agent' && this.cfg.gitSync && heldForClash(this, d.key, rel)) { refused.push(`${rel} (held for a clash: propose with quilt_clash)`); for (const t of types) undo.add(t); continue }
       if (this.mayWrite(a, rel)) continue
       refused.push(rel)
       for (const t of types) undo.add(t)
@@ -1603,20 +1603,6 @@ class Room {
    * field and find it on their next look), and the relay checks GitHub itself for a branch no
    * folder can bring in. Returns whether any branch follows it.
    */
-  /** The open clashes on every branch (relay-clash.js): timeouts and hand-offs, about once a minute, one branch at a time in its queue. */
-  async reviewClashTick () {
-    if (!this.cfg.gitSync || !this.mirrors || !this.meta.sync) return []
-    const out = []
-    for (const [key, st] of Object.entries(this.meta.sync)) {
-      if (!st || !Array.isArray(st.held) || !st.held.some((h) => h.why === 'clash')) continue
-      const rec = upstreamsOf(this)[key]
-      const repo = rec && parseRepo(`https://${rec.repo}`)
-      if (!repo) continue
-      out.push(...await this.mirrors.queue(repo.slug, rec.ref, () => reviewClashes(this, key)))
-    }
-    return out
-  }
-
   /** Whether session `other` has this session's owner in it (its owner or a member): then the app may offer to open it ("Open the live session", plan §8). */
   sameOwner (other) {
     if (!other || !this.meta.ownerSub) return false
@@ -1667,7 +1653,7 @@ class Room {
     const want = String(repo).toLowerCase().replace(/^github\.com\//, '')
     const follows = Object.values(upstreamsOf(this)).some((rec) => rec && rec.repo && String(rec.repo).toLowerCase().replace(/^github\.com\//, '') === want)
     if (!follows) return false
-    // Checks on a commit the relay pushed with merged files in it (relay-clash.js): failing pulls both parties in; passing lets held files go.
+    // Checks on a commit the relay pushed with merged files in it (relay-clash.js): failing tells both parties; nothing is held.
     if (this.cfg.gitSync && this.mirrors && /^[0-9a-f]{40}$/.test(sha)) {
       for (const [key, rec] of Object.entries(upstreamsOf(this))) {
         if (!rec || String(rec.repo).toLowerCase().replace(/^github\.com\//, '') !== want || (branch && rec.ref !== branch && !(rec.ref && branch === `quilt/live/${rec.ref}`))) continue
@@ -2868,16 +2854,11 @@ class Room {
             .then((r) => { if (this.conns.has(ws)) send(ws, jsonMessage(MSG_MEMBERS, { members: this.memberList(), sessionName: this.meta.name || '', admitBy: this.admitBy, pending: this.personCanAdmit(this.access.get(ws)) ? this.pendingList() : [], reply: r })) })
           return
         }
-        // A member's offline edits clash with the session's (plan §7.1): held and merged as a clash (relay-clash.js).
+        // Sent by apps before 0.3.99 for offline edits that clashed with the session's: Quilt merges those on its own now
+        // (session.js autoSettle), so there is nothing to hold. Answered ok so an older app doesn't retry.
         if (req.op === 'offlineClash') {
-          const id = req.id
-          const a = this.access.get(ws)
-          if (!a || (a.state && a.state !== 'approved') || !ws.branch || a.role === 'viewer') throw new Error('not in this session')
-          if (!this.cfg.gitSync || !this.mirrors) throw new Error('This session doesn\'t sync live with GitHub.')
-          const text = (v) => typeof v === 'string' && v.length <= 1024 * 1024 ? v : null
-          openOfflineClash(this, ws.branch, { who: { name: a.name }, path: String(req.path || ''), mergeId: typeof req.mergeId === 'string' ? req.mergeId.slice(0, 40) : null, base: text(req.base), ours: text(req.ours), oursDeleted: req.oursDeleted === true, binary: req.binary === true })
-            .then((h) => ({ id, ok: true, offlineClash: { task: h.task, merger: h.merger, reviewer: h.reviewer } }), (err) => ({ id, ok: false, error: err.message }))
-            .then((r) => { this.broadcastBranches(); if (this.conns.has(ws)) send(ws, jsonMessage(MSG_MEMBERS, { members: this.memberList(), sessionName: this.meta.name || '', admitBy: this.admitBy, pending: this.personCanAdmit(this.access.get(ws)) ? this.pendingList() : [], reply: r })) })
+          const r = { id: req.id, ok: true, offlineClash: { task: null, merger: null, reviewer: null } }
+          if (this.conns.has(ws)) send(ws, jsonMessage(MSG_MEMBERS, { members: this.memberList(), sessionName: this.meta.name || '', admitBy: this.admitBy, pending: this.personCanAdmit(this.access.get(ws)) ? this.pendingList() : [], reply: r }))
           return
         }
         // A person adopts an agent's branch into theirs (addendum 16.7; relay-outbound.js adoptBranch).
@@ -2896,7 +2877,6 @@ class Room {
           const id = req.id
           const a = this.access.get(ws)
           if (!a || (a.state && a.state !== 'approved') || !ws.branch) throw new Error('not in this session')
-          if (!this.cfg.gitSync || !this.mirrors) throw new Error('This session doesn\'t sync live with GitHub, so there are no held clashes here.')
           clashAction(this, { who: { name: a.name, kind: a.kind === 'agent' ? 'agent' : 'human', access: a }, key: ws.branch, action: String(req.action || ''), path: String(req.path || ''), content: typeof req.content === 'string' ? req.content : null, checks: req.checks && typeof req.checks === 'object' ? req.checks : null, force: req.force === true })
             .then((out) => out.ok ? { id, ok: true, clash: out, text: out.text } : { id, ok: false, error: out.text }, (err) => ({ id, ok: false, error: err.message }))
             .then((r) => { if (this.conns.has(ws)) send(ws, jsonMessage(MSG_MEMBERS, { members: this.memberList(), sessionName: this.meta.name || '', admitBy: this.admitBy, pending: this.personCanAdmit(this.access.get(ws)) ? this.pendingList() : [], reply: r })) })
@@ -3042,6 +3022,11 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
   // Without sign-in the limit is per address, and the message says so (as before).
   const TOO_MANY = passKey ? 'too many new sessions; try again later' : 'too many new sessions from this address; try again later'
   if (dataDir) fs.mkdirSync(dataDir, { recursive: true })
+  // Without git http-backend every agent's git request would fail with a 500: agents' folders use GitHub's git instead.
+  if (cfg.gitSync && cfg.gitHttp && passKey && !gitHttpBackendWorks(cfg.gitHttpCommand)) {
+    cfg.gitHttp = false
+    log('⚠️ this machine\'s git has no http-backend (on Alpine it comes with git-daemon): agents\' folders can\'t use git through the relay, so they use GitHub\'s')
+  }
   // The relay's git mirrors (relay-git.js): one bare repository per GitHub repository, one writer per branch.
   const mirrors = mirrorManager({ dataDir, log, ...(cfg.pushGapMs !== undefined ? { pushGapMs: cfg.pushGapMs } : {}), ...(cfg.pushesPerHour !== undefined ? { pushesPerHour: cfg.pushesPerHour } : {}) })
   // One live session per repository and branch (relay-lease.js): in the accounts API when this relay has one, else in this process.
@@ -3337,14 +3322,30 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
     return n
   }
   /** How busy and how big a loaded room is (relay-metrics.js): for /stats and the accounts API's placement. */
-  const roomStats = (room) => ({ room: room.name, conns: room.conns.size, hosted: room.hostedSeen.size, ...room.meter.lastMinute(), docBytes: room.totalBytes(), diskBytes: diskOf(room.name) })
-  const machineStats = () => {
+  // Git pushes into a room's view (/git/<room>.git) being received now.
+  const gitPushes = new Map() // room -> count
+  /** Git work running or queued for a room now: pushes being received, and commits, pushes and bring-ins on its branches (relay-git.js). */
+  const gitBusy = (room) => {
+    let n = gitPushes.get(room.name) || 0
+    for (const rec of Object.values((room.meta && room.meta.upstreams) || {})) {
+      const repo = rec && rec.repo && rec.ref ? parseRepo(`https://${rec.repo}`) : null
+      if (repo) n += mirrors.queue.pending(repo.slug, rec.ref)
+    }
+    return n
+  }
+  const roomStats = (room) => ({ room: room.name, conns: room.conns.size, hosted: room.hostedSeen.size, ...room.meter.lastMinute(), docBytes: room.totalBytes(), diskBytes: diskOf(room.name), git: gitBusy(room) })
+  // The machine's own load (memory, event-loop lag, CPU), a minute at a time: for scaling the cluster on load.
+  const machineMeter = new MachineMeter()
+  /** The machine: rooms loaded, connections, memory, lag, CPU and every room's traffic (`list`: the rooms' stats, when already made). */
+  const machineStats = (list = null) => {
     const mem = process.memoryUsage()
-    return { rooms: rooms.size, conns: wss.clients.size, rss: mem.rss, heapUsed: mem.heapUsed, load: os.loadavg()[0], uptime: Math.round(process.uptime()) }
+    const traffic = list || [...rooms.values()].filter((r) => !r.ended).map((r) => r.meter.lastMinute())
+    return { rooms: rooms.size, conns: wss.clients.size, rss: mem.rss, heapUsed: mem.heapUsed, load: os.loadavg()[0], uptime: Math.round(process.uptime()), ...machineLoad({ meter: machineMeter, rooms: traffic, mem }) }
   }
   const stats = () => {
     for (const name of diskBytes.keys()) if (!rooms.has(name)) diskBytes.delete(name)
-    return { machine: machineStats(), rooms: [...rooms.values()].filter((r) => !r.ended).map(roomStats) }
+    const list = [...rooms.values()].filter((r) => !r.ended).map(roomStats)
+    return { machine: machineStats(list), rooms: list }
   }
   /** Closes a loaded room on this machine without ending it: its people reconnect (to wherever it is now). */
   const unloadForMove = (name) => {
@@ -3519,13 +3520,22 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
       })
     }
     // An org or workspace changed what agents may commit (the accounts API): every open session asks again now.
+    // With { room }, only that session (one just put in or taken out of a workspace), if this machine holds it.
     if (url.pathname === '/github/policy' && req.method === 'POST') {
       const auth = String(req.headers.authorization || '').replace(/^Bearer\s+/i, '')
       const want = cfg.relayApiSecret || ''
       const same = want && auth.length === want.length && crypto.timingSafeEqual(Buffer.from(auth), Buffer.from(want))
       if (!same) return json(401, { error: 'only the accounts API may say this' })
-      for (const room of rooms.values()) room.refreshPolicy()
-      return json(200, { ok: true, rooms: rooms.size })
+      return readJson(req, 4 * 1024, (err, b) => {
+        const one = !err && b && typeof b.room === 'string' && b.room ? b.room : ''
+        if (one) {
+          const room = ROOM_RE.test(one) ? rooms.get(one) : null
+          if (room) room.refreshPolicy()
+          return json(200, { ok: true, rooms: room ? 1 : 0 })
+        }
+        for (const room of rooms.values()) room.refreshPolicy()
+        json(200, { ok: true, rooms: rooms.size })
+      })
     }
     // Someone connected GitHub (the API's callback): every session they are in checks its repository again now.
     if (url.pathname === '/github/connected' && req.method === 'POST') {
@@ -3559,7 +3569,14 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
       if (req.method !== 'POST') return text(405, 'method not allowed')
       return readJson(req, 4096, (err, b) => {
         if (err) return json(400, { error: err.message })
-        if (action === 'release') { letRoomGo(name); return json(200, { ok: true }) }
+        if (action === 'release') {
+          // Never mid git push or commit: the API tries again later (or another room).
+          const loaded = rooms.get(name)
+          const busy = loaded ? gitBusy(loaded) : gitPushes.get(name) || 0
+          if (busy) return json(409, { error: `${name} has git work running (${busy}); try again later` })
+          letRoomGo(name)
+          return json(200, { ok: true })
+        }
         if (action === 'restore') { cluster.restore(name); log(`cluster: ${name} stays on this machine`); return json(200, { ok: true }) }
         if (action === 'forget') { forgetRoomHere(name, String((b && b.to) || '')); return json(200, { ok: true }) }
         // fetch
@@ -3618,7 +3635,7 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
     // branches at /git/<room>.git, signed in with the member's Quilt pass (git's password; quilt git-credential).
     const gm = url.pathname.match(/^\/git\/([A-Za-z0-9_-]{1,64})\.git(\/info\/refs|\/git-upload-pack|\/git-receive-pack)$/)
     if (gm) {
-      if (!cfg.gitSync || !passKey || !dataDir) return text(404, 'this relay serves no git')
+      if (!cfg.gitSync || !cfg.gitHttp || !passKey || !dataDir) return text(404, 'this relay serves no git')
       const name = gm[1]
       const basic = /^Basic\s+([A-Za-z0-9+/=]+)$/i.exec(String(req.headers.authorization || ''))
       const decoded = basic ? Buffer.from(basic[1], 'base64').toString('utf8') : ''
@@ -3629,6 +3646,11 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
       if (!room.letIn(pass)) return text(403, NOT_LET_IN)
       const service = gm[2] === '/info/refs' ? url.searchParams.get('service') : gm[2].slice(1)
       if (service === 'git-receive-pack') { const a = room.httpAccess(pass); if (a && a.role === 'viewer') return text(403, 'you can only view this session, so you can\'t push') }
+      if (gm[2] === '/git-receive-pack') {
+        // A push being received: the room is not moved to another machine until it is done.
+        gitPushes.set(name, (gitPushes.get(name) || 0) + 1)
+        res.once('close', () => { const n = (gitPushes.get(name) || 1) - 1; if (n > 0) gitPushes.set(name, n); else gitPushes.delete(name) })
+      }
       const def = viewRepoOf(room)
       if (!def) { log(`[${name}] git over http: no GitHub repository (default ${room.defaultKey}: ${JSON.stringify(upstreamsOf(room)[room.defaultKey] || null)})`); return text(404, 'this session follows no GitHub repository') }
       const repo = parseRepo(`https://${def.repo}`)
@@ -3637,7 +3659,7 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
         if (!(await runGit(m.dir, ['rev-parse', '--verify', '-q', `refs/remotes/origin/${def.ref}`])).out.trim()) await room.mirrors.fetch(m.dir, def.ref, { token: await room.gitToken(repo, { access: 'read' }) }).catch(() => {})
         const dir = await ensureView(dataDir, name, m.dir)
         await syncViewRefs(room, dir, def.repo)
-        serveGit(req, res, { root: path.dirname(dir), pathInfo: `/${name}.git${gm[2]}`, query: url.search.slice(1), env: { QUILT_HOOK_URL: `http://127.0.0.1:${httpServer.address().port}/git-hook`, QUILT_HOOK_TOKEN: hookToken, QUILT_ROOM: name, QUILT_WHO: `${pass.kind}:${pass.sub}`, QUILT_WHO_NAME: String(pass.name || '') } })
+        serveGit(req, res, { root: path.dirname(dir), pathInfo: `/${name}.git${gm[2]}`, query: url.search.slice(1), env: { QUILT_HOOK_URL: `http://127.0.0.1:${httpServer.address().port}/git-hook`, QUILT_HOOK_TOKEN: hookToken, QUILT_ROOM: name, QUILT_WHO: `${pass.kind}:${pass.sub}`, QUILT_WHO_NAME: String(pass.name || '') }, git: cfg.gitHttpCommand, onError: (why) => log(`[${name}] git over http (${req.method} ${gm[2]}): ${why}`) })
       })().catch((err) => { log(`[${name}] git over http: ${err.message}`); if (!res.headersSent) text(500, 'the relay could not serve git right now') })
       return
     }
@@ -3683,7 +3705,9 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
           return { index: { rooms: [...idx.rooms] }, noteHosted: (o) => cluster.hostedRoom(account, o), shareWebhook: (w) => cluster.hostedWebhook(account, w || null) }
         })
         : Promise.resolve({})
-      return shared.then((more) => handleHostedMcp({ req, res, pass, workspaces, body: parsed, relay: { getRoom, roomEnded, refused, hosted, saveHosted, webhooks, log, endedMessage: ENDED_MESSAGE, updates, visitStart: hostedVisitStart, visitEnd: hostedVisitEnd, ...more } })
+      // Secrets' names, through the accounts API as the agent (its pass).
+      const secrets = cfg.apiUrl && pass.kind === 'agent' ? { apiUrl: cfg.apiUrl, pass: String(req.headers['x-quilt-pass']), fetch: apiFetch } : null
+      return shared.then((more) => handleHostedMcp({ req, res, pass, workspaces, secrets, body: parsed, relay: { getRoom, roomEnded, refused, hosted, saveHosted, webhooks, log, endedMessage: ENDED_MESSAGE, updates, visitStart: hostedVisitStart, visitEnd: hostedVisitEnd, ...more } })
         // Its first session on this machine: the webhook it set elsewhere applies here too.
         .then(() => { if (more.index && cluster.hosted.get(account)) applySharedWebhook(account, cluster.hosted.get(account).webhook) }))
         .catch((err) => { log(`mcp error: ${err.message}`); if (!res.headersSent) text(500, 'mcp error') })
@@ -3915,7 +3939,9 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
       }
     })
     reportTicker = setInterval(() => {
-      cluster.report(machineStats(), [...rooms.values()].filter((r) => !r.ended).map(roomStats)).catch((err) => log(`cluster: could not report this machine's minute: ${err.message}`))
+      machineMeter.roll()
+      const list = [...rooms.values()].filter((r) => !r.ended).map(roomStats)
+      cluster.report(machineStats(list), list).catch((err) => log(`cluster: could not report this machine's minute: ${err.message}`))
       // Hosted agents in loaded rooms: a webhook set or removed on another machine reaches this one within a minute.
       let n = 0
       for (const [id, h] of hosted) if (n < 200 && h.rooms && Object.keys(h.rooms).some((r) => rooms.has(r))) { n++; syncHosted(id) }
@@ -3965,7 +3991,7 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
   sweeper.unref()
   // Branches only hosted agents work on, with no folder online: the relay looks for new commits on GitHub itself.
   const upstreamTicker = cfg.upstreamAuto && cfg.upstreamCheckMs > 0
-    ? setInterval(() => { for (const room of rooms.values()) if (!room.ended) { room.checkUpstreams().catch((err) => log(`[${room.name}] could not look for new commits: ${err.message}`)); room.reviewClashTick().catch((err) => log(`[${room.name}] could not look at the clashes: ${err.message}`)) } }, Math.min(cfg.gitSync ? 20 * 1000 : 60 * 1000, cfg.upstreamCheckMs)) // with the mirror, often enough that each branch's minute look isn't a tick late
+    ? setInterval(() => { for (const room of rooms.values()) if (!room.ended) { room.checkUpstreams().catch((err) => log(`[${room.name}] could not look for new commits: ${err.message}`)) } }, Math.min(cfg.gitSync ? 20 * 1000 : 60 * 1000, cfg.upstreamCheckMs)) // with the mirror, often enough that each branch's minute look isn't a tick late
     : null
   upstreamTicker?.unref()
   // Live-sync leases renewed, and let go of for sessions that ended or sat idle for a day (relay-lease.js).
@@ -4009,6 +4035,7 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
         cluster, // exposed for tests
         close: async () => {
           clearInterval(reportTicker)
+          machineMeter.stop()
           if (cluster) cluster.stop()
           clearInterval(leaseTicker)
           clearInterval(heartbeat)