heyquilt 0.0.1 → 0.3.90

package/src/relay-turn-sync.js

@@ -0,0 +1,307 @@
+// The relay's side of `quilt sync` (turn-sync.js): a sandboxed agent's folder synced in one shot,
+// over the hosted MCP (the quilt_sync tool, relay-mcp.js), with the sync state kept here rather
+// than in the folder. Each of the agent's sandboxes (a slot, from its QUILT_AGENT_KEY) has a
+// checkpoint per branch: what its folder held at the end of its last sync, texts included, so the
+// next sync can tell who changed what and merge. The rules are turn-sync-plan.js.
+//
+// Steps, one request each: plan (the folder's manifest in; what to upload, pull, remove and keep
+// aside out), push (contents in; written, deleted or a merge record, decided again against the
+// session as it is now), pull (the session's version of some paths), done (the folder's manifest
+// after writing; the checkpoint saved).
+import fs from 'node:fs'
+import path from 'node:path'
+import zlib from 'node:zlib'
+import crypto from 'node:crypto'
+import * as Y from 'yjs'
+import { applyTextDiff } from './textdiff.js'
+import { isSafeRelPath, LARGE_FILE_BYTES } from './pathrules.js'
+import { openMerge, MAX_RECORD_TEXT } from './merges.js'
+import { decide, planSync, nextCheckpoint, mergeTexts, behind, sessionEntry, SKIP } from './turn-sync-plan.js'
+
+export const SLOT = /^[A-Za-z0-9_-]{8,64}$/
+const HASH = /^[0-9a-f]{40}$/
+/** A text pushed by quilt sync: at most this long (as quilt_write_file). */
+export const MAX_SYNC_TEXT = 1024 * 1024
+/** One pull answer carries at most this much file content; the rest is asked for again. */
+export const MAX_PULL_BYTES = 8 * 1024 * 1024
+/** Checkpoints unused this long are forgotten. */
+const KEEP_MS = 90 * 24 * 60 * 60 * 1000
+export const ROOM_KEY = '∅room' // a room without branch documents
+
+const sha1 = (x) => crypto.createHash('sha1').update(x).digest('hex')
+
+/**
+ * Checkpoints, per room, agent and slot: { [branch]: { epoch, at, files } }. On disk under
+ * <dataDir>/turn-sync/<room>/ (gzipped JSON, one file per agent and slot), in memory without one.
+ */
+export class CheckpointStore {
+  constructor ({ dir = null } = {}) {
+    this.dir = dir
+    this.mem = new Map()
+  }
+
+  file (account, slot) { return path.join(this.dir, `${sha1(`${account}\0${slot}`)}.json.gz`) }
+
+  get (account, slot) {
+    const k = `${account}\0${slot}`
+    if (this.mem.has(k)) return this.mem.get(k)
+    let v = {}
+    if (this.dir) {
+      try { v = JSON.parse(zlib.gunzipSync(fs.readFileSync(this.file(account, slot))).toString('utf8')) || {} } catch {}
+    }
+    this.mem.set(k, v)
+    return v
+  }
+
+  put (account, slot, value) {
+    this.mem.set(`${account}\0${slot}`, value)
+    if (!this.dir) return
+    fs.mkdirSync(this.dir, { recursive: true })
+    const f = this.file(account, slot)
+    fs.writeFileSync(`${f}.tmp`, zlib.gzipSync(JSON.stringify(value)))
+    fs.renameSync(`${f}.tmp`, f)
+    this.prune()
+  }
+
+  /** Forgets checkpoints nobody synced against for KEEP_MS. */
+  prune (now = Date.now()) {
+    if (!this.dir) return
+    let names = []
+    try { names = fs.readdirSync(this.dir) } catch { return }
+    for (const n of names) {
+      const f = path.join(this.dir, n)
+      try { if (now - fs.statSync(f).mtimeMs > KEEP_MS) fs.rmSync(f, { force: true }) } catch {}
+    }
+  }
+}
+
+/** The room's checkpoint store, made the first time it is needed. */
+export function checkpointsOf (room) {
+  if (!room.turnSync) room.turnSync = new CheckpointStore({ dir: room.dataDirOf ? path.join(room.dataDirOf, 'turn-sync', room.name) : null })
+  return room.turnSync
+}
+
+/** Removes a room's checkpoints (the room's data is being removed). */
+export function dropCheckpoints (dataDir, room) {
+  if (dataDir) fs.rmSync(path.join(dataDir, 'turn-sync', room), { recursive: true, force: true })
+}
+
+/** The session's files on a branch: path -> { h, text } (text files and small binaries) or { h, stored }. */
+export function sessionState (files, blobs) {
+  const out = new Map()
+  for (const [rel, t] of files) {
+    if (!isSafeRelPath(rel) || !t) continue
+    const text = t.toString()
+    out.set(rel, sessionEntry(sha1(text), { text }))
+  }
+  for (const [rel, b] of blobs) {
+    if (!isSafeRelPath(rel) || out.has(rel) || !b || !HASH.test(String(b.hash || ''))) continue
+    out.set(rel, b.stored ? { h: b.hash, stored: true } : sessionEntry(b.hash))
+  }
+  return out
+}
+
+/** The folder's manifest from a request: { path: sha1 | 'skip' }, safe paths only. */
+export function cleanManifest (files) {
+  const out = {}
+  for (const [rel, h] of Object.entries(files || {})) {
+    if (!isSafeRelPath(rel)) continue
+    if (h === SKIP || HASH.test(String(h))) out[rel] = h
+  }
+  return out
+}
+
+/** A file's content sent in a push: { buf, text, hash } or { gone: true }; null when malformed or too big. */
+function content (v) {
+  if (!v || typeof v !== 'object') return null
+  if (v.deleted === true) return { gone: true }
+  if (typeof v.text === 'string') {
+    const buf = Buffer.from(v.text, 'utf8')
+    if (buf.length > MAX_SYNC_TEXT) return null
+    return { text: v.text, buf, hash: sha1(buf) }
+  }
+  if (typeof v.base64 === 'string') {
+    const buf = Buffer.from(v.base64, 'base64')
+    if (buf.length >= LARGE_FILE_BYTES) return null
+    return { buf, hash: sha1(buf) }
+  }
+  return null
+}
+
+/**
+ * One step of a sync. `args`: { step, slot, files?, paths? }. `ctx`: { room, fdoc, files, blobs,
+ * branchKey, epoch, account, me, origin, globalRefusal(), pathRefusal(rel) -> { kind: 'aside' |
+ * 'claimed', why, by? } | null, record({ rel, kind, before, after, detail }), lastEditor(rel), ghosts? }.
+ * `ghosts`: the paths someone else has boxed (docs/superpowers/specs/2026-10-16-git-boxes-design.md): their
+ * unfinished work never reaches this folder. The folder sees each as it last had it (its checkpoint), or as it
+ * has it now; an edit to one is refused like one to a held file, and the folder's version comes back.
+ * Returns the answer (a plain object, sent as JSON).
+ */
+export function turnSyncStep (args, ctx) {
+  const slot = String(args.slot || '')
+  if (!SLOT.test(slot)) throw new Error('slot must be 8 to 64 letters, numbers, dashes or underscores (from your QUILT_AGENT_KEY)')
+  const store = checkpointsOf(ctx.room)
+  const all = store.get(ctx.account, slot)
+  const saved = all[ctx.branchKey]
+  // A checkpoint of an older document of this branch (removed and started again) says nothing about this one.
+  const checkpoint = saved && (saved.epoch || null) === (ctx.epoch || null) ? saved : null
+  const ghosts = ctx.ghosts instanceof Set ? ctx.ghosts : new Set()
+  const deleted = (rel) => !ghosts.has(rel) && !ctx.files.has(rel) && !ctx.blobs.has(rel) && (ctx.files._map.has(rel) || ctx.blobs._map.has(rel))
+  const cpFiles = (checkpoint && checkpoint.files) || {}
+  // The session as this folder may see it: someone else's box shows as the folder's own version.
+  const session = (local = null) => {
+    const out = sessionState(ctx.files, ctx.blobs)
+    for (const rel of ghosts) {
+      const cp = cpFiles[rel]
+      if (cp && HASH.test(String(cp.h || ''))) out.set(rel, sessionEntry(cp.h, { text: typeof cp.t === 'string' ? cp.t : null }))
+      else if (local && local[rel] && local[rel] !== SKIP) out.set(rel, sessionEntry(local[rel]))
+      else out.delete(rel)
+    }
+    return out
+  }
+  const step = args.step
+
+  if (step === 'plan') {
+    const local = cleanManifest(args.files)
+    const p = planSync({ local, session: session(local), checkpoint, deleted })
+    const refused = p.upload.length || p.delete.length ? ctx.globalRefusal() : null
+    return { step, ...p, ...(refused ? { refused } : {}) }
+  }
+
+  if (step === 'push') {
+    const out = { step, pushed: [], deleted: [], merged: [], merges: [], pull: [], remove: [], aside: [], kept: [], skipped: [] }
+    const refusedAll = ctx.globalRefusal()
+    const first = !checkpoint
+    const base = (checkpoint && checkpoint.files) || {}
+    // Each path is only changed once, after it was looked at: one look at the session is enough.
+    const state = session()
+    for (const [rel, v] of Object.entries(args.files || {})) {
+      if (!isSafeRelPath(rel)) continue
+      const c = content(v)
+      if (!c) { out.skipped.push(rel); continue }
+      // Someone else's box this folder never synced: left as it is here, and not taken into the session.
+      if (ghosts.has(rel) && !cpFiles[rel]) { const why = ctx.pathRefusal(rel); out.kept.push({ path: rel, why: why ? why.why : `${rel} is someone else's unfinished work` }); continue }
+      const now = state.get(rel) || null
+      const d = decide({ local: c.gone ? null : c.hash, session: now, base: base[rel] || null, first, deleted: deleted(rel) })
+      if (d === 'same') continue
+      if (d === 'skip') { out.skipped.push(rel); continue }
+      if (d === 'pull' || d === 'remove') { out[d].push(rel); continue }
+      if (d === 'aside-pull' || d === 'aside-remove') { out.aside.push(rel); out[d.slice(6)].push(rel); continue }
+      // What follows changes the session: refused as a whole (an unanswered message, a full session), or for this file.
+      if (refusedAll) { out.kept.push({ path: rel, why: refusedAll }); continue }
+      const why = ctx.pathRefusal(rel)
+      if (why && why.kind === 'keep') { out.kept.push({ path: rel, why: why.why }); continue }
+      if (why || d === 'merge') {
+        settle(rel, c, now, base[rel] || null, why, ctx, out)
+        continue
+      }
+      if (d === 'push') { write(rel, c, ctx); out.pushed.push(rel) } else if (d === 'delete') { remove(rel, ctx); out.deleted.push(rel) }
+    }
+    return out
+  }
+
+  if (step === 'pull') {
+    const now = session()
+    const files = {}
+    const more = []
+    let bytes = 0
+    for (const rel of Array.isArray(args.paths) ? args.paths : []) {
+      if (typeof rel !== 'string' || !isSafeRelPath(rel)) continue
+      const s = now.get(rel)
+      if (bytes >= MAX_PULL_BYTES) { more.push(rel); continue }
+      // Someone else's box: the folder's own last version when it has the text, else nothing (the folder keeps what it has).
+      if (ghosts.has(rel)) { if (s && typeof s.text === 'string') { files[rel] = { text: s.text, h: s.h }; bytes += s.text.length } continue }
+      if (!s) { files[rel] = { gone: true }; continue }
+      if (s.stored) { files[rel] = { stored: true }; continue }
+      if (typeof s.text === 'string') { files[rel] = { text: s.text, h: s.h }; bytes += s.text.length; continue }
+      const b = ctx.blobs.get(rel)
+      files[rel] = { base64: b.data || '', h: s.h }
+      bytes += (b.data || '').length
+    }
+    return { step, files, more }
+  }
+
+  if (step === 'done') {
+    const local = cleanManifest(args.files)
+    const now = session(local)
+    const next = nextCheckpoint(checkpoint, local, now)
+    store.put(ctx.account, slot, { ...all, [ctx.branchKey]: { epoch: ctx.epoch || null, ...next } })
+    return { step, saved: Object.keys(next.files).length, behind: behind(local, now).slice(0, 200) }
+  }
+
+  throw new Error('step must be plan, push, pull or done')
+}
+
+/** Writes the folder's version of rel into the session, as the agent. */
+function write (rel, c, ctx) {
+  ctx.fdoc.transact(() => {
+    const existed = ctx.files.has(rel) || ctx.blobs.has(rel)
+    const before = ctx.files.get(rel)?.toString() ?? ''
+    if (typeof c.text === 'string') {
+      ctx.blobs.delete(rel)
+      let t = ctx.files.get(rel)
+      if (!t) { t = new Y.Text(); ctx.files.set(rel, t) }
+      const detail = applyTextDiff(t, c.text)
+      ctx.record({ rel, kind: existed ? 'edited' : 'created', before, after: c.text, detail })
+    } else {
+      ctx.files.delete(rel)
+      ctx.blobs.set(rel, { hash: c.hash, data: c.buf.toString('base64') })
+      ctx.record({ rel, kind: existed ? 'edited' : 'created', detail: `${c.buf.length} bytes` })
+    }
+  }, ctx.origin)
+}
+
+/** Deletes rel from the session, as the agent. */
+function remove (rel, ctx) {
+  ctx.fdoc.transact(() => {
+    const before = ctx.files.get(rel)?.toString()
+    ctx.files.delete(rel)
+    ctx.blobs.delete(rel)
+    ctx.record({ rel, kind: 'deleted', ...(typeof before === 'string' ? { before, after: '' } : {}), detail: '' })
+  }, ctx.origin)
+}
+
+/**
+ * Both changed rel (or it isn't ours to change): merged when the lines don't overlap and nothing
+ * stands in the way; otherwise a merge record (as Session.openConflict: the session's version
+ * stays, ours is in the record) and the folder takes the session's version with ours kept aside.
+ */
+function settle (rel, c, now, base, why, ctx, out) {
+  const theirs = now && typeof now.text === 'string' ? now.text : null
+  if (!why && typeof c.text === 'string' && theirs !== null && base && typeof base.t === 'string') {
+    const merged = mergeTexts(base.t, c.text, theirs)
+    if (merged !== null) {
+      write(rel, { text: merged }, ctx)
+      out.merged.push(rel)
+      if (sha1(Buffer.from(merged, 'utf8')) !== c.hash) out.pull.push(rel)
+      return
+    }
+  }
+  if (why && why.kind === 'aside') {
+    // Not ours to change (a viewer, a folder outside our scope): the session's version comes back.
+    out.aside.push(rel)
+    out[now ? 'pull' : 'remove'].push(rel)
+    out.kept.push({ path: rel, why: why.why })
+    return
+  }
+  const binary = typeof c.text !== 'string' && !c.gone
+  const textOf = (t) => (typeof t === 'string' && t.length <= MAX_RECORD_TEXT ? t : null)
+  const theirsKey = now ? (typeof now.text === 'string' ? now.text : `bin:${now.h}`) : null
+  const by = ctx.lastEditor ? ctx.lastEditor(rel) : null
+  const rec = openMerge(ctx.fdoc, ctx.fdoc.getMap('merges'), {
+    path: rel,
+    by: ctx.me,
+    byId: ctx.account,
+    others: by && by !== ctx.me ? [by] : [],
+    kind: why && why.kind === 'claimed' ? 'claimed' : 'conflict',
+    ours: binary || c.gone ? null : textOf(c.text),
+    oursDeleted: !!c.gone,
+    base: binary ? null : textOf(base && base.t),
+    theirsHash: theirsKey === null ? null : sha1(theirsKey),
+    binary,
+    claimedBy: why && why.kind === 'claimed' ? why.by || null : null
+  }, ctx.origin)
+  out.merges.push({ path: rel, id: rec.id, kind: rec.kind })
+  if (!c.gone) out.aside.push(rel)
+  out[now ? 'pull' : 'remove'].push(rel)
+}