@agentshouse/kit 0.2.1-c8179f06883b6 → 0.2.1-c83d468784cc9

package/dist/copies.js

@@ -1,20 +1,37 @@
 import { parse } from '@agentshouse/mdmodel';
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { mkdir, readFile, readdir, realpath, rename, rm, rmdir, stat, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { finalRefusal, HouseRefusal, retrying } from './api.js';
-import { blobId, blobText, git, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
+import { chunkTexts, stagedChunks } from './chunks.js';
+import { blobId, blobText, git, isText, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
+import { paused } from './files.js';
 import { kitHome } from './home.js';
 import { operationId } from './operation.js';
 import { refusalOf } from './refusals.js';
 import { changed } from './translate.js';
-import { relayed } from './relay.js';
 const BRANCH = 'main';
 const PROVENANCE = '_provenance';
-const IDENTITY = { GIT_COMMITTER_NAME: 'House Kit', GIT_COMMITTER_EMAIL: 'house-kit@localhost' };
+const IDENTITY = { GIT_COMMITTER_NAME: 'Kit', GIT_COMMITTER_EMAIL: 'house-kit@localhost' };
+const ROOMS = 'rooms/';
+const ENUMERATE = '/kit/door/changes/enumerate';
+const NOT_READY = 'not ready yet; Kit copies it once House has built it';
+const STAGED_CHUNKS = 'house-chunks';
+// A Room answered not ready is asked again on this schedule: its chunk build usually catches up within seconds, so the first asks come soon and the last repeats until it does.
+const ASKED_AGAIN_MS = [1000, 2000, 3000, 5000, 8000, 11_000, 15_000];
+class NotReady extends Error {
+    rooms;
+    constructor(rooms) {
+        super(NOT_READY);
+        this.rooms = rooms;
+    }
+}
 function causeOf(error) {
     return error instanceof Error ? error.message : String(error);
 }
+function houseCause(error) {
+    return error instanceof HouseRefusal ? (refusalOf(error.text)?.text ?? causeOf(error)) : causeOf(error);
+}
 function logged(error) {
     process.stderr.write(`kit: ${causeOf(error)}\n`);
 }
@@ -29,6 +46,13 @@ const REBASED = new Set([
 function rebaseFixes(refusal) {
     return refusal.code === 'edit_conflict' && refusal.conflicts.some((code) => REBASED.has(code));
 }
+function textOnly(paths) {
+    const [named, them] = paths.length === 1 ? [`${paths[0]} is`, 'it'] : [`${paths.join(', ')} are`, 'them'];
+    return `${named} not text, and house push sends text only; remove ${them} from the commit and place ${them} with house upload`;
+}
+function outside(path) {
+    return { refused: true, text: `${path} is not a House Local copy; run house push inside one` };
+}
 function short(commit) {
     return commit.slice(0, 12);
 }
@@ -48,6 +72,9 @@ function expiredAuthority(error) {
         return undefined;
     }
 }
+function grouped(items, size) {
+    return items.flatMap((_, index) => (index % size === 0 ? [items.slice(index, index + size)] : []));
+}
 function covers(path, prefix) {
     return prefix === '' || path === prefix || path.startsWith(`${prefix}/`);
 }
@@ -56,6 +83,12 @@ function sidecar(path) {
     segments.splice(-1, 0, PROVENANCE);
     return segments.join('/');
 }
+function recordOf(key) {
+    return join(kitHome(), 'local-copies', `${key}.json`);
+}
+function stagingPrefix() {
+    return `.${createHash('sha256').update(kitHome()).digest('hex').slice(0, 16)}-`;
+}
 function rootOf(handle) {
     return `/rooms/${handle}`;
 }
@@ -83,32 +116,59 @@ async function removeFile(repository, path) {
             return;
     }
 }
-async function readOnlyPlaced(repository, held, written, removed) {
+async function readOnlyFiles(repository, held) {
     const tracked = new Set((await git(repository, ['ls-files', '-z'])).split('\0'));
     const holding = new Set(held);
-    for (const path of removed) {
-        if (!holding.has(path))
-            continue;
-        if (!tracked.has(path))
-            await removeFile(repository, path);
-        holding.delete(path);
-    }
-    for (const [path, content] of written) {
-        if (tracked.has(path))
-            continue;
-        if (!holding.has(path) && (await exists(join(repository, path))))
-            continue;
-        await writeText(join(repository, path), content);
-        holding.add(path);
-    }
-    return [...holding].sort();
+    return {
+        remove: async (path) => {
+            if (!holding.has(path))
+                return;
+            if (!tracked.has(path))
+                await removeFile(repository, path);
+            holding.delete(path);
+        },
+        place: async (path, content) => {
+            if (tracked.has(path))
+                return;
+            if (!holding.has(path) && (await exists(join(repository, path))))
+                return;
+            await writeText(join(repository, path), content);
+            holding.add(path);
+        },
+        held: () => [...holding].sort(),
+    };
+}
+async function readOnlyPlaced(repository, held, written, removed) {
+    const files = await readOnlyFiles(repository, held);
+    for (const path of removed)
+        await files.remove(path);
+    for (const [path, content] of written)
+        await files.place(path, content);
+    return files.held();
+}
+async function importedWhole(repository, scope, texts, commit, readOnly) {
+    const revisions = {};
+    async function* tracked() {
+        for await (const { path, content, revision: held, provenance } of texts) {
+            if (declaredWritable(path, scope)) {
+                revisions[path] = held;
+                yield [path, content];
+            }
+            else {
+                await readOnly.place(path, content);
+            }
+            if (provenance !== undefined)
+                await readOnly.place(sidecar(path), provenance);
+        }
+    }
+    return { house: await importCommits(repository, [{ ...commit, files: tracked() }]), revisions };
 }
 async function ruled(path, rules) {
     if ((await readFile(path, 'utf8').catch(() => null)) !== rules)
         await writeFile(path, rules);
 }
 async function ignoreRules(repository, writable, copy) {
-    const readOnly = ('read_only' in copy ? copy.read_only : [...copy.readOnly.keys()]).filter((path) => !path.split('/').includes(PROVENANCE));
+    const readOnly = copy.read_only.filter((path) => !path.split('/').includes(PROVENANCE));
     const protectedPath = (path) => copy.protected.some((prefix) => prefix !== '' && covers(path, prefix));
     const whole = copy.protected.filter((prefix) => prefix !== '' && !writable.some((path) => covers(path, prefix)));
     const house = [
@@ -117,7 +177,7 @@ async function ignoreRules(repository, writable, copy) {
     ].map(pattern);
     const granted = readOnly.filter((path) => !protectedPath(path)).map(pattern);
     await ruled(join(repository, '.gitignore'), [
-        '# House Kit writes this file and .ignore. The paths below are read-only House',
+        '# Kit writes this file and .ignore. The paths below are read-only House',
         '# content: Git does not track them, and House refuses a change to them even',
         '# when this file is edited or a path is force-added.',
         '/.gitignore',
@@ -132,7 +192,7 @@ async function ignoreRules(repository, writable, copy) {
         '',
     ].join('\n'));
     await ruled(join(repository, '.ignore'), [
-        '# House Kit writes this file so native search still reads the read-only House',
+        '# Kit writes this file so native search still reads the read-only House',
         '# content that .gitignore keeps out of Git.',
         `!${PROVENANCE}`,
         ...[...house, ...granted].map((rule) => `!${rule}`),
@@ -168,6 +228,7 @@ export class LocalCopies {
     house;
     copies = new Map();
     queues = new Map();
+    waiting = new Set();
     answering = Promise.resolve();
     constructor(house) {
         this.house = house;
@@ -175,7 +236,7 @@ export class LocalCopies {
             key: 'owner',
             submit: async (operation, changes) => {
                 try {
-                    const outcome = await retrying(() => house.post('/kit/door/batch', { operation_id: operation, changes }));
+                    const outcome = await retrying(() => house.post('/kit/door/files/batch', { operation_id: operation, changes }));
                     return { accepted: outcome.contents };
                 }
                 catch (error) {
@@ -199,70 +260,129 @@ export class LocalCopies {
             this.copies.set(copy.key, copy);
         }
     }
+    async swept() {
+        const rooms = join(localCopies(), 'rooms');
+        for (const name of await readdir(rooms).catch(() => [])) {
+            if (name.startsWith(stagingPrefix()))
+                await rm(join(rooms, name), { recursive: true, force: true });
+        }
+    }
     async mapping() {
         await this.answering;
         return [...this.copies.values()]
-            .filter((copy) => copy.active)
+            .filter((copy) => copy.unfollowed === undefined)
             .map((copy) => `${rootOf(copy.room_handle)}: ${copy.repository}`)
             .sort();
     }
-    async connected() {
-        const copied = await this.answered(() => this.fetched(false));
-        if (copied.refused)
-            logged(copied.text);
+    connected() {
+        return this.selected();
+    }
+    async changed(rooms) {
+        if (rooms.length > 0)
+            await this.selected(rooms);
     }
     fetch() {
         return this.answered(() => this.fetched(true));
     }
+    async selected(rooms) {
+        const copied = await this.answered(() => this.fetched(false, rooms));
+        if (copied.refused)
+            logged(copied.text);
+    }
     answered(work) {
         const answered = this.answering.then(work);
         this.answering = answered.catch(() => undefined);
         return answered;
     }
-    async fetched(refreshing) {
-        const { local_copy: answer } = await this.house.post('/kit/local-copy/selection', {});
-        const synced = new Map(answer.rooms.map((room) => [`rooms/${room.room_ref}`, room.room_handle]));
-        return this.holding([...new Set([...this.copies.keys(), ...synced.keys()])].sort(), async () => {
+    async fetched(refreshing, rooms) {
+        const { local_copy: answer } = await this.house.post('/kit/local-copies/selection/read', rooms === undefined ? {} : { rooms });
+        const synced = new Map(answer.rooms.map((room) => [`${ROOMS}${room.room_ref}`, room.room_handle]));
+        const named = rooms === undefined ? null : new Set(rooms.map((room) => `${ROOMS}${room}`));
+        const held = () => [...this.copies.values()].filter((copy) => named === null || named.has(copy.key));
+        return this.holding([...new Set([...held().map((copy) => copy.key), ...synced.keys()])].sort(), async () => {
             const said = [];
             let refused = false;
-            for (const copy of this.copies.values()) {
-                if (synced.has(copy.key) || !copy.active)
+            const mark = refreshing ? 'said' : 'unsaid';
+            for (const copy of held()) {
+                if (synced.has(copy.key) || copy.unfollowed === 'said' || copy.unfollowed === mark)
                     continue;
-                await this.save({ ...copy, active: false });
-                said.push(`${rootOf(copy.room_handle)}: no longer synced to this Environment; its files stay`);
+                await this.save({ ...copy, unfollowed: mark });
+                if (refreshing)
+                    said.push(`${rootOf(copy.room_handle)}: no longer followed; kept as it is`);
             }
             const current = [];
+            const wanted = new Map();
             for (const [key, handle] of synced) {
                 try {
                     const held = this.copies.get(key);
                     if (held === undefined || !(await exists(held.repository))) {
-                        said.push(await this.materialize(key, handle));
+                        wanted.set(key, handle);
                         continue;
                     }
-                    const copy = held.room_handle === handle ? held : await this.save({ ...held, room_handle: handle });
-                    if (copy.active)
-                        current.push(copy);
-                    else
-                        said.push(...moved(copy, await this.save(await this.received({ ...copy, active: true }))));
+                    const { unfollowed, ...followed } = held;
+                    current.push(unfollowed === undefined && held.room_handle === handle ? held : await this.save({ ...followed, room_handle: handle }));
                 }
                 catch (error) {
                     refused = true;
                     said.push(`${rootOf(handle)}: ${causeOf(error)}`);
                 }
             }
-            if (refreshing && current.length > 0) {
-                try {
-                    const refreshed = await this.refreshed(current);
-                    current.forEach((copy, index) => said.push(...moved(copy, refreshed[index])));
+            const unready = [];
+            const unsettled = [];
+            if (wanted.size > 0) {
+                const { authorities } = await this.discovered([...wanted.values()]);
+                for (const [key, handle] of wanted) {
+                    const authority = authorities.find((entry) => `${ROOMS}${entry.room_ref}` === key);
+                    try {
+                        if (authority === undefined)
+                            throw new Error('House serves nothing of it to copy');
+                        said.push(await this.copied({ key, handle, authority }));
+                    }
+                    catch (error) {
+                        if (error instanceof NotReady) {
+                            unready.push(...error.rooms);
+                            said.push(`${rootOf(handle)}: ${NOT_READY}`);
+                        }
+                        else {
+                            refused = true;
+                            unsettled.push(key.slice(ROOMS.length));
+                            said.push(`${rootOf(handle)}: ${causeOf(error)}`);
+                        }
+                    }
                 }
-                catch (error) {
+            }
+            const reading = refreshing ? current : current.filter((copy) => this.waiting.has(copy.room_ref));
+            if (reading.length > 0) {
+                const failed = await this.refreshed(reading, unready).then(() => null, houseCause);
+                for (const copy of reading) {
+                    said.push(...(unready.includes(copy.room_ref) ? [`${rootOf(copy.room_handle)}: ${NOT_READY}`] : moved(copy, this.copies.get(copy.key))));
+                }
+                if (failed !== null) {
                     refused = true;
-                    said.push(error instanceof HouseRefusal ? (refusalOf(error.text)?.text ?? causeOf(error)) : causeOf(error));
+                    unsettled.push(...reading.map((copy) => copy.room_ref));
+                    said.push(failed);
                 }
             }
+            for (const room of rooms ?? [...this.waiting]) {
+                if (!unready.includes(room) && !unsettled.includes(room))
+                    this.waiting.delete(room);
+            }
+            const due = unready.filter((room) => !this.waiting.has(room));
+            for (const room of due)
+                this.waiting.add(room);
+            if (due.length > 0)
+                void this.askedAgain(due);
             return { refused, text: said.length === 0 ? 'Nothing new from House.' : said.join('\n') };
         });
     }
+    async askedAgain(rooms) {
+        let asking = rooms;
+        for (let step = 0; asking.length > 0; step++) {
+            await paused(ASKED_AGAIN_MS[Math.min(step, ASKED_AGAIN_MS.length - 1)]);
+            await this.selected().catch(logged);
+            asking = asking.filter((room) => this.waiting.has(room));
+        }
+    }
     async observed(report) {
         const key = await this.keyAt(dirname(report.common));
         if (key === undefined)
@@ -291,11 +411,11 @@ export class LocalCopies {
     async push(cwd, selector, caller) {
         const located = await run(cwd, ['rev-parse', '--path-format=absolute', '--git-common-dir', '--show-toplevel']);
         if (located.status !== 0)
-            return { refused: true, text: `${cwd} is not a House Local copy; run house push inside one` };
+            return outside(cwd);
         const [common, worktree] = located.stdout.toString('utf8').split('\n');
         const key = await this.keyAt(dirname(common));
         if (key === undefined)
-            return { refused: true, text: `${worktree} is not a House Local copy; run house push inside one` };
+            return outside(worktree);
         return this.exclusive(key, () => this.pushed(key, worktree, selector, caller));
     }
     async keyAt(repository) {
@@ -316,7 +436,7 @@ export class LocalCopies {
         return key === undefined ? work() : this.exclusive(key, () => this.holding(rest, work));
     }
     async save(copy) {
-        const path = join(kitHome(), 'local-copies', `${copy.key}.json`);
+        const path = recordOf(copy.key);
         await mkdir(dirname(path), { recursive: true, mode: 0o700 });
         const temporary = `${path}.${process.pid}`;
         await writeFile(temporary, JSON.stringify(copy), { mode: 0o600 });
@@ -324,128 +444,170 @@ export class LocalCopies {
         this.copies.set(copy.key, copy);
         return copy;
     }
-    async read(handle) {
-        const prefixes = [rootOf(handle)];
-        const discovery = await this.house.post('/kit/door/discover', { prefixes });
-        let bundle = await this.house.post('/kit/door/bootstrap', { prefixes });
-        if ('transfer' in bundle)
-            bundle = (await (await fetch(relayed(bundle.transfer.url))).json());
-        const [view] = bundle.scopes;
-        if (view === undefined)
-            return null;
-        const authority = discovery.authorities.find((entry) => entry.room_ref === view.manifest.anchor);
-        const contents = new Map(view.contents.map((entry) => [entry.path, entry.content]));
-        const read = {
-            anchor: view.manifest.anchor,
-            position: view.manifest.position,
-            log_epoch: view.manifest.log_epoch,
-            write: authority.operations.write,
-            protected: authority.protected_paths,
-            writable: new Map(),
-            revisions: {},
-            readOnly: new Map(),
-        };
-        for (const file of view.manifest.files) {
-            const content = contents.get(file.path);
-            if (file.writable) {
-                read.writable.set(file.path, content);
-                read.revisions[file.path] = file.revision;
-            }
-            else {
-                read.readOnly.set(file.path, content);
+    async discovered(handles) {
+        const discover = (group) => this.house.post('/kit/door/rooms/discover', { prefixes: group.map(rootOf) });
+        const [first, ...rest] = handles;
+        const { bounds, authorities } = await discover([first]);
+        for (const group of grouped(rest, bounds.max_prefixes))
+            authorities.push(...(await discover(group)).authorities);
+        return { bounds, authorities };
+    }
+    async chunked(root, room, directory) {
+        await mkdir(directory, { recursive: true });
+        for (let renewed = false;; renewed = true) {
+            const { rooms } = await this.house.post('/kit/door/room-chunks/list', { prefixes: [root] });
+            const list = rooms.find((entry) => entry.room_ref === room);
+            if (list === undefined)
+                throw new Error('House serves nothing of it to copy');
+            if (!list.ready)
+                throw new NotReady([room]);
+            const held = new Set(await readdir(directory));
+            const failure = await stagedChunks(directory, list.chunks.filter(({ hash }) => !held.has(hash)));
+            if (failure === undefined) {
+                return { position: list.position, log_epoch: list.log_epoch, texts: chunkTexts(directory, list.chunks, list.models) };
             }
-            if (file.provenance !== undefined)
-                read.readOnly.set(sidecar(file.path), file.provenance);
+            if (renewed && (await readdir(directory)).length === held.size)
+                throw failure;
         }
-        return read;
     }
-    async materialize(key, handle) {
-        const read = await this.read(handle);
-        if (read === null)
-            throw new Error('House serves nothing of it to copy');
-        const repository = join(localCopies(), 'rooms', read.anchor);
+    async materialized({ key, handle, authority }, whole) {
+        const repository = join(localCopies(), 'rooms', authority.room_ref);
         if (await exists(repository))
-            throw new Error(`${repository} already exists and House Kit leaves it as it is`);
-        const staging = join(dirname(repository), `.${randomUUID()}`);
+            throw new Error(`${repository} already exists and Kit leaves it as it is`);
+        const staging = join(dirname(repository), `${stagingPrefix()}${randomUUID()}`);
+        const scope = { write: authority.operations.write, protected: authority.protected_paths };
         await mkdir(staging, { recursive: true });
-        let house;
-        let readOnly;
+        let copy;
         try {
             await git(staging, ['init', '-q', '-b', BRANCH]);
-            house = await importCommits(staging, [
-                { message: `House ${rootOf(handle)}`, from: null, files: read.writable },
-            ]);
+            const read = await whole(join(staging, '.git', STAGED_CHUNKS));
+            const readOnly = await readOnlyFiles(staging, []);
+            const { house, revisions } = await importedWhole(staging, scope, read.texts, { message: `House ${rootOf(handle)}`, from: null }, readOnly);
             await git(staging, ['update-ref', `refs/heads/${BRANCH}`, house]);
             await git(staging, ['reset', '-q', '--hard']);
-            readOnly = await readOnlyPlaced(staging, [], read.readOnly, []);
-            await ignoreRules(staging, [...read.writable.keys()], read);
+            copy = {
+                key,
+                room_ref: authority.room_ref,
+                room_handle: handle,
+                repository,
+                house,
+                position: read.position,
+                log_epoch: read.log_epoch,
+                ...scope,
+                read_only: readOnly.held(),
+                manifests: { [house]: { parent: null, files: revisions } },
+                moves: [],
+            };
+            await ignoreRules(staging, Object.keys(revisions), copy);
+            await rm(join(staging, '.git', STAGED_CHUNKS), { recursive: true, force: true });
             await rename(staging, repository);
         }
         catch (error) {
             await rm(staging, { recursive: true, force: true });
             throw error;
         }
-        await this.save({
-            key,
-            room_ref: read.anchor,
-            room_handle: handle,
-            repository,
-            active: true,
-            house,
-            position: read.position,
-            log_epoch: read.log_epoch,
-            write: read.write,
-            protected: read.protected,
-            read_only: readOnly,
-            manifests: { [house]: { parent: null, files: read.revisions } },
-            moves: [],
-        });
-        return `${rootOf(handle)}: copied to ${repository}`;
+        return this.save(copy);
     }
-    async received(copy) {
-        const read = await this.read(copy.room_handle);
-        if (read === null)
-            return copy;
-        const house = await importCommits(copy.repository, [
-            { message: `House ${rootOf(copy.room_handle)} at position ${read.position}`, from: copy.house, replaces: true, files: read.writable },
-        ]);
-        const removed = copy.read_only.filter((path) => !read.readOnly.has(path));
-        const readOnly = await readOnlyPlaced(copy.repository, copy.read_only, read.readOnly, removed);
-        const next = {
-            ...copy,
-            house,
-            position: read.position,
-            log_epoch: read.log_epoch,
-            write: read.write,
-            protected: read.protected,
-            read_only: readOnly,
-            manifests: { ...copy.manifests, [house]: { parent: null, files: read.revisions } },
-        };
-        await ignoreRules(copy.repository, [...read.writable.keys()], next);
-        return next;
+    async copied(wanted) {
+        const { handle, authority } = wanted;
+        const root = rootOf(handle);
+        let answer = null;
+        try {
+            if (!this.waiting.has(authority.room_ref)) {
+                answer = await this.house.post(ENUMERATE, {
+                    prefixes: [root],
+                    positions: [
+                        { authority: authority.room_ref, position: '0', log_epoch: authority.log_epoch, page: { through: authority.position, read: 0 } },
+                    ],
+                });
+            }
+        }
+        catch (error) {
+            if (expiredAuthority(error) !== authority.room_ref)
+                throw error;
+        }
+        const view = answer?.authorities.find((entry) => entry.authority === authority.room_ref);
+        if (answer !== null && view === undefined)
+            throw new Error('House serves nothing of it to copy');
+        if (view !== undefined && view.next === undefined) {
+            const texts = view.changes.filter((change) => !('removed' in change));
+            const copy = await this.materialized(wanted, async () => ({ position: view.position, log_epoch: view.log_epoch, texts }));
+            return `${root}: copied to ${copy.repository}`;
+        }
+        const copy = await this.materialized(wanted, (directory) => this.chunked(root, authority.room_ref, directory));
+        const [caught] = await this.caughtUp([copy], true);
+        await succeeded(copy.repository, ['merge', '--ff-only', '-q', caught.house]);
+        return `${root}: copied to ${copy.repository}`;
     }
-    async refreshed(copies) {
-        const discovery = await this.house.post('/kit/door/discover', {
-            prefixes: copies.map((copy) => rootOf(copy.room_handle)),
-        });
-        const regranted = new Map();
+    async received(copy, authority) {
+        const root = rootOf(copy.room_handle);
+        const directory = join(copy.repository, '.git', STAGED_CHUNKS);
+        const scope = { write: authority.operations.write, protected: authority.protected_paths };
+        try {
+            const read = await this.chunked(root, copy.room_ref, directory);
+            const readOnly = await readOnlyFiles(copy.repository, copy.read_only);
+            for (const path of copy.read_only)
+                await readOnly.remove(path);
+            const commit = { message: `House ${root} at position ${read.position}`, from: copy.house, replaces: true };
+            const { house, revisions } = await importedWhole(copy.repository, scope, read.texts, commit, readOnly);
+            const next = {
+                ...copy,
+                ...scope,
+                house,
+                position: read.position,
+                log_epoch: read.log_epoch,
+                read_only: readOnly.held(),
+                manifests: { ...copy.manifests, [house]: { parent: null, files: revisions } },
+            };
+            await ignoreRules(copy.repository, Object.keys(revisions), next);
+            return next;
+        }
+        finally {
+            await rm(directory, { recursive: true, force: true });
+        }
+    }
+    async reread(copy) {
+        const { authorities } = await this.discovered([copy.room_handle]);
+        const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
+        if (authority === undefined)
+            throw new NotReady([copy.room_ref]);
+        return this.received(copy, authority);
+    }
+    async refreshed(copies, unready) {
+        const { bounds, authorities } = await this.discovered(copies.map((copy) => copy.room_handle));
+        const read = [];
+        const unchanged = [];
         for (const copy of copies) {
-            const authority = discovery.authorities.find((entry) => entry.room_ref === copy.room_ref);
+            const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
             if (authority !== undefined &&
                 JSON.stringify([authority.operations.write, authority.protected_paths]) !== JSON.stringify([copy.write, copy.protected])) {
-                regranted.set(copy.key, await this.save(await this.received(copy)));
+                try {
+                    read.push(await this.save(await this.received(copy, authority)));
+                }
+                catch (error) {
+                    if (!(error instanceof NotReady))
+                        throw error;
+                    unready.push(...error.rooms);
+                }
+            }
+            else {
+                unchanged.push(copy);
             }
         }
-        const caught = await this.caughtUp(copies.filter((copy) => !regranted.has(copy.key)), false);
-        return copies.map((copy) => regranted.get(copy.key) ?? caught.find((held) => held.key === copy.key));
+        for (const group of grouped(unchanged, bounds.max_prefixes))
+            await this.caughtUp(group, false, unready);
+        for (const group of grouped(read, bounds.max_prefixes))
+            await this.caughtUp(group, true, unready);
     }
-    async caughtUp(copies, whole) {
+    async caughtUp(copies, whole, unready) {
         const caught = new Map(copies.map((copy) => [copy.key, copy]));
+        const read = new Set();
+        const waiting = unready ?? [];
         let asking = [...copies];
         while (asking.length > 0) {
             let answer;
             try {
-                answer = await this.house.post('/kit/door/enumerate', {
+                answer = await this.house.post(ENUMERATE, {
                     prefixes: asking.map((copy) => rootOf(copy.room_handle)),
                     positions: asking.map((copy) => ({ authority: copy.room_ref, position: copy.position, log_epoch: copy.log_epoch })),
                 });
@@ -454,8 +616,22 @@ export class LocalCopies {
                 const expired = asking.find((copy) => copy.room_ref === expiredAuthority(error));
                 if (expired === undefined)
                     throw error;
-                caught.set(expired.key, await this.save(await this.received(expired)));
                 asking = asking.filter((copy) => copy !== expired);
+                if (read.has(expired.key)) {
+                    waiting.push(expired.room_ref);
+                    continue;
+                }
+                read.add(expired.key);
+                try {
+                    const next = await this.save(await this.reread(expired));
+                    caught.set(expired.key, next);
+                    asking.push(next);
+                }
+                catch (refused) {
+                    if (!(refused instanceof NotReady))
+                        throw refused;
+                    waiting.push(expired.room_ref);
+                }
                 continue;
             }
             const advanced = [];
@@ -467,8 +643,10 @@ export class LocalCopies {
                 caught.set(copy.key, next);
                 advanced.push(next);
             }
-            asking = whole ? advanced : [];
+            asking = advanced.filter((copy) => whole || read.has(copy.key));
         }
+        if (unready === undefined && waiting.length > 0)
+            throw new NotReady(waiting);
         return copies.map((copy) => caught.get(copy.key));
     }
     async incorporated(copy, changes, position, logEpoch) {
@@ -532,9 +710,6 @@ export class LocalCopies {
             }
             copy = await this.save(without(copy));
         }
-        if (!copy.active) {
-            return { refused: true, text: `${rootOf(copy.room_handle)} is no longer synced to this Environment, so House takes no push from it` };
-        }
         const listed = (await git(copy.repository, ['rev-list', '--topo-order', commit])).split('\n');
         const base = listed.find((ancestor) => ancestor in copy.manifests);
         if (base === undefined) {
@@ -545,7 +720,7 @@ export class LocalCopies {
             changes = await this.derived(copy, base, commit);
         }
         catch (error) {
-            return { refused: true, text: causeOf(error) };
+            return { refused: true, text: houseCause(error) };
         }
         if (changes.length === 0)
             return { refused: false, text: `Nothing to submit: ${short(commit)} matches House.` };
@@ -572,7 +747,18 @@ export class LocalCopies {
         const after = await tree(copy.repository, commit);
         const moved = movedPaths(copy.moves, before, after);
         const arrived = new Set(moved.values());
-        const text = (entry, path) => blobText(copy.repository, entry, path);
+        const arriving = [...after].filter(([path, entry]) => before.get(path) !== entry);
+        if (arriving.length > 0) {
+            const { bounds } = await this.house.post('/kit/door/rooms/discover', { prefixes: [root] });
+            const binary = [];
+            for (const [path, entry] of arriving) {
+                if (!(await isText(copy.repository, entry, bounds.max_authored_file_bytes)))
+                    binary.push(path);
+            }
+            if (binary.length > 0)
+                throw new Error(textOnly(binary));
+        }
+        const text = (entry) => blobText(copy.repository, entry);
         const changes = [];
         for (const path of [...new Set([...before.keys(), ...after.keys()])].sort()) {
             if (arrived.has(path))
@@ -585,7 +771,7 @@ export class LocalCopies {
                 changes.push({ op: 'rename', path: `${root}/${path}`, to: `${root}/${to}`, base: known });
                 const moving = after.get(to);
                 if (moving !== prior) {
-                    changes.push({ op: 'replace', path: `${root}/${to}`, base: known, content: await text(moving, to) });
+                    changes.push({ op: 'replace', path: `${root}/${to}`, base: known, content: await text(moving) });
                 }
                 continue;
             }
@@ -594,9 +780,9 @@ export class LocalCopies {
             if (next === undefined)
                 changes.push({ op: 'remove', path: `${root}/${path}`, base: known });
             else if (prior === undefined)
-                changes.push({ op: 'create', path: `${root}/${path}`, content: await text(next, path) });
+                changes.push({ op: 'create', path: `${root}/${path}`, content: await text(next) });
             else
-                changes.push(...changed(`${root}/${path}`, path, known, await text(prior, path), await text(next, path)));
+                changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next)));
         }
         return changes;
     }
@@ -625,8 +811,10 @@ export class LocalCopies {
             }
             if (outcome.refused.code === 'expired')
                 return this.recovered(copy);
-            const rebased = rebaseFixes(outcome.refused);
             let settled = await this.save(without(copy));
+            if (settled.unfollowed !== undefined)
+                return { refused: true, text: outcome.refused.text };
+            const rebased = rebaseFixes(outcome.refused);
             try {
                 settled = (await this.caughtUp([settled], true))[0];
             }
@@ -663,7 +851,7 @@ export class LocalCopies {
             written.push({
                 path: `${root}/${path}`,
                 revision: entry === undefined ? null : (revisionAt(current, current.house, path) ?? null),
-                content: entry === undefined ? null : await blobText(copy.repository, entry, path),
+                content: entry === undefined ? null : await blobText(copy.repository, entry),
             });
         }
         return this.finished(await this.save({ ...current, push: { ...push, state: 'accepted', written } }));
@@ -707,7 +895,7 @@ export class LocalCopies {
                     {
                         path: `${rootOf(copy.room_handle)}/${path}`,
                         revision: entry === undefined ? null : (revisionAt(copy, copy.house, path) ?? null),
-                        content: entry === undefined ? null : await blobText(copy.repository, entry, path),
+                        content: entry === undefined ? null : await blobText(copy.repository, entry),
                     },
                 ];
             }
@@ -752,7 +940,7 @@ export class LocalCopies {
     async landed(repository, changes, house, selected) {
         const text = async (path) => {
             const entry = house.get(relative(path));
-            return entry === undefined ? undefined : blobText(repository, entry, path);
+            return entry === undefined ? undefined : blobText(repository, entry);
         };
         for (const change of changes) {
             const current = await text(change.path);
@@ -778,7 +966,7 @@ export class LocalCopies {
         return true;
     }
     async revisionText(path, wanted) {
-        const page = await this.house.post('/kit/door/revisions', {
+        const page = await this.house.post('/kit/door/files/revisions/list', {
             path,
             revision: wanted,
         });