@agentshouse/mdruntime 0.1.0-c0bed82db8d73 → 0.1.0-c0c9861edaba9

package/dist/connection.js

@@ -1,5 +1,5 @@
 import { escapePointer, parseFrontmatter, roomTarget, target } from '@agentshouse/mdmodel';
-import { attachedModel, NO_ROOM_TYPES, roomTypesOf } from './attached-model.js';
+import { attachedModel, builtInModel, roomTypesOf } from './attached-model.js';
 import { cached, modelTypes, NO_MODEL, sameModel, } from './model.js';
 import { observing, whole } from './observation.js';
 import { filterOf, identityOf, planOf, plain, queryKey, } from './query.js';
@@ -8,7 +8,7 @@ import { seededResult } from './seed.js';
 import { documentKey, roomOfKey } from './rows.js';
 import { searchPlanOf } from './search.js';
 import { capped, MAXIMUM_WINDOW } from './windows.js';
-import { bootstrapRoom } from './bootstrap.js';
+import { bootstrapRoom, NOTHING_DONE } from './bootstrap.js';
 import { changeSet, mergeChangeSets } from './change-set.js';
 import { createConversationStore, } from './conversation.js';
 import { leaving as leavingIn, partnerOf, queuedFile, } from './change.js';
@@ -54,7 +54,10 @@ const CONVERSATION_LANE = 'conversations';
 const ROOM_READS = 256;
 const ROOM_ATTEMPTS = 16;
 const ERASE_BATCH = 512;
-const REDERIVE_BATCH = 256;
+const DERIVE_BYTES = 1 << 23;
+const DERIVE_FLOOR = 1 << 18;
+// An index build aims at this much database work, so a Save, a query or the next part's progress waits behind at most one short build.
+const DERIVE_AIM_MS = 250;
 const WALK_BYTES = 1 << 20;
 const QUERY_BYTES = 1 << 20;
 const STAGE_BYTES = 1 << 20;
@@ -62,14 +65,9 @@ const STAGE_FLOOR = 1 << 16;
 // A batch aims at this much database work, so a Save or a query waits behind at most one short write.
 const STAGE_AIM_MS = 50;
 const PARTS_HELD = 4;
+const QUEUED_BYTES = 1 << 23;
 // Bytes still arriving are reported at most this often, well inside the 2 s progress bound, so readiness is not rebuilt for every chunk the network hands over.
 const ARRIVAL_REPORT_MS = 250;
-const INSTALL_STEPS = [
-    'writing',
-    'downloading',
-    'replaying',
-    'listing',
-];
 const CLEAN_CHANGES = 2_000;
 // A Room that locks offline stays readable this long after the connection drops, so a laptop waking or changing network keeps working while a disconnected device locks soon after.
 const LOCK_GRACE_MS = 120_000;
@@ -114,9 +112,9 @@ export function composeRuntime(options) {
     const rooms = new FencedMap(isFenced);
     const counted = new Map();
     const unreadable = new Set();
-    const isModelFenced = (key) => isFenced(key.slice(key.indexOf(' ') + 1));
-    const models = new FencedMap(isModelFenced);
-    const liveModels = new FencedMap(isModelFenced);
+    const builtIn = builtInModel(roomTypes);
+    const models = new FencedMap(isFenced);
+    const liveModels = new FencedMap(isFenced);
     const invalidModels = new Set();
     const discovered = new Map();
     const reads = doorReads(options.adapter, (locator) => discovered.get(locator), () => generation);
@@ -124,23 +122,19 @@ export function composeRuntime(options) {
     const scopedRefusals = new Map();
     const sourceRefusals = new Map();
     const declarationRefusals = new Set();
-    function modelKey(locator, attached) {
-        return `${attached} ${locator}`;
+    function modelOf(locator) {
+        return models.get(locator) ?? NO_MODEL;
     }
-    function modelOf(locator, attached = 'collection') {
-        return models.get(modelKey(locator, attached)) ?? NO_MODEL;
-    }
-    function savedModel(locator, attached) {
-        return invalidModels.has(modelKey(locator, attached))
+    function savedModel(locator) {
+        return invalidModels.has(locator)
             ? { status: 'unavailable', reason: 'revoked' }
-            : modelOf(locator, attached);
+            : modelOf(locator);
+    }
+    function composed(record) {
+        return compositionOf(record, builtIn);
     }
     function compositionFor(record) {
-        return compositionOf({
-            ...record,
-            model: savedModel(record.locator, 'collection'),
-            selectionModel: savedModel(record.locator, 'typed-selection'),
-        });
+        return composed({ ...record, model: savedModel(record.locator) });
     }
     let refusal = null;
     let storageLoss = null;
@@ -218,13 +212,27 @@ export function composeRuntime(options) {
     const catching = new Set();
     const transfers = new FencedMap(isFenced);
     const installs = new Map();
+    const keptParts = new Map();
     const arrivals = new Map();
+    const underived = new Map();
     let arrivalReported = 0;
     let onScreen = options.shell.onScreen;
     const shownWaiters = [];
     let partsFree = PARTS_HELD;
+    let queuedBytes = 0;
     const partWaiters = [];
-    let stageBytes = STAGE_FLOOR * 4;
+    const stageBudget = {
+        bytes: STAGE_FLOOR * 4,
+        floor: STAGE_FLOOR,
+        ceiling: STAGE_BYTES,
+        aim: STAGE_AIM_MS,
+    };
+    const deriveBudget = {
+        bytes: DERIVE_FLOOR * 4,
+        floor: DERIVE_FLOOR,
+        ceiling: DERIVE_BYTES,
+        aim: DERIVE_AIM_MS,
+    };
     let installWrites = Promise.resolve();
     const conversations = createConversationStore({
         agents: declaredAgents,
@@ -276,7 +284,7 @@ export function composeRuntime(options) {
             return undefined;
         const local = await openLocal();
         const slice = await stepped(slicing(await local.entries(held.record.key, 'visible', paths), paths));
-        return { ...slice, collection: collectionOf(slice.documents, compositionOf(held.record)) };
+        return { ...slice, collection: collectionOf(slice.documents, composed(held.record)) };
     }
     function roomDraftOf(record, visible) {
         return structuredClone({
@@ -483,7 +491,7 @@ export function composeRuntime(options) {
             const governed = governs(reach, room);
             const attached = [
                 ...(governed.collection ? [modelOf(room.locator)] : []),
-                ...(governed.selection ? [modelOf(room.locator, 'typed-selection')] : []),
+                ...(governed.selection ? [builtIn] : []),
             ];
             return {
                 locator: room.locator,
@@ -567,7 +575,10 @@ export function composeRuntime(options) {
             if (held === undefined || closed || held.indexed)
                 return;
             const key = held.record.key;
-            const done = await exclusive(async () => (await openLocal()).rederive(key, REDERIVE_BATCH));
+            const done = await exclusive(async () => {
+                const local = await openLocal();
+                return timed(deriveBudget, () => local.rederive(key, deriveBudget.bytes));
+            });
             const now = replicas.get(locator);
             if (now?.record.key !== key)
                 continue;
@@ -610,7 +621,11 @@ export function composeRuntime(options) {
         })();
     }
     function roomsOfReach(reach) {
-        return new Set([...reach.rooms, ...reach.documents.map(roomOfKey)]);
+        return new Set([
+            ...reach.rooms,
+            ...reach.documents.map(roomOfKey),
+            ...reach.areas.flatMap((area) => (area.types.length === 0 ? [] : [area.room])),
+        ]);
     }
     function reachedRooms(plan) {
         const reached = plan.reach === null ? null : roomsOfReach(plan.reach);
@@ -729,7 +744,9 @@ export function composeRuntime(options) {
         const parts = { done: 0, total: 0 };
         const bytes = { done: 0, total: 0 };
         const received = { done: 0, total: 0 };
-        const steps = new Set();
+        let installing = 0;
+        let unlisted = false;
+        let listed = false;
         let waiting = false;
         for (const { room, state } of coverage) {
             if (state === 'unadmitted')
@@ -738,30 +755,40 @@ export function composeRuntime(options) {
             if (state === 'complete')
                 rooms.done += 1;
             const transfer = transfers.get(room);
-            const held = installs.get(room);
-            const inventory = transfer?.inventory ?? replicas.get(room)?.record.inventory ?? null;
+            const replica = replicas.get(room)?.record.inventory ?? null;
+            const inventory = transfer?.inventory ?? replica ?? installs.get(room) ?? null;
+            const underway = state === 'acquiring' || (installs.has(room) && state !== 'complete');
+            if (underway)
+                installing += 1;
+            if (underway && inventory === null)
+                unlisted = true;
             if (inventory !== null) {
-                const done = transfer?.done ?? inventory;
-                const share = held?.share ?? null;
+                listed = true;
+                const done = transfer?.done ?? (replica === null ? NOTHING_DONE : inventory);
                 parts.total += inventory.parts;
                 bytes.total += inventory.bytes;
                 parts.done += done.parts;
                 bytes.done += done.bytes;
-                if (share !== null && share.entries > 0) {
-                    bytes.done += Math.floor((share.bytes * share.written) / share.entries);
-                }
                 received.total += inventory.bytes;
-                received.done += done.bytes + (share?.bytes ?? 0);
-                for (const arrived of arrivals.get(room)?.values() ?? [])
-                    received.done += arrived;
+                received.done += done.bytes;
+                for (const held of [keptParts.get(room), arrivals.get(room)]) {
+                    for (const arrived of held?.values() ?? [])
+                        received.done += arrived;
+                }
             }
-            const installing = held !== undefined && state !== 'complete';
-            if (installing)
-                steps.add(held.step);
-            waiting ||= installing || state === 'acquiring';
+            waiting ||= underway;
         }
+        const step = installing === 0
+            ? null
+            : !listed
+                ? 'listing'
+                : unlisted || received.done < received.total
+                    ? 'downloading'
+                    : bytes.done < bytes.total
+                        ? 'writing'
+                        : 'replaying';
         return {
-            step: INSTALL_STEPS.find((step) => steps.has(step)) ?? null,
+            step,
             paused: waiting && !onScreen ? 'hidden' : null,
             rooms,
             parts,
@@ -1029,7 +1056,7 @@ export function composeRuntime(options) {
                     const records = [...replicas].map(([locator, held]) => ({
                         locator,
                         held,
-                        record: { ...held.record, model: revoked, selectionModel: revoked },
+                        record: { ...held.record, model: revoked },
                     }));
                     let answer = null;
                     try {
@@ -1039,18 +1066,14 @@ export function composeRuntime(options) {
                                 locator,
                                 key: record.key,
                                 record: storedRecord(record),
-                                composition: compositionOf(record),
+                                composition: composed(record),
                             })),
                         });
                     }
                     catch {
                         journal.record('model revocation persistence failed');
                     }
-                    const keys = new Set(models.keys());
-                    for (const locator of rooms.keys()) {
-                        keys.add(modelKey(locator, 'collection'));
-                        keys.add(modelKey(locator, 'typed-selection'));
-                    }
+                    const keys = new Set([...models.keys(), ...rooms.keys()]);
                     for (const key of keys) {
                         invalidModels.add(key);
                         models.set(key, revoked);
@@ -1105,7 +1128,7 @@ export function composeRuntime(options) {
                 ? null
                 : {
                     roots: selection.roots,
-                    model: modelTypes(modelOf(room.locator, 'typed-selection')),
+                    model: modelTypes(builtIn),
                 },
             providerOwnedPaths: room.providerOwnedPaths,
             roomVersion: replicas.get(room.locator)?.record.roomVersion ?? null,
@@ -1194,7 +1217,7 @@ export function composeRuntime(options) {
             queuesWithoutReplica.delete(locator);
             rooms.delete(locator);
             roomDrafts.delete(locator);
-            transfers.delete(locator);
+            untransferred(locator);
             catching.delete(locator);
             discovered.delete(locator);
             announced.delete(locator);
@@ -1205,12 +1228,9 @@ export function composeRuntime(options) {
             scopedRefusals.delete(locator);
             sourceRefusals.delete(locator);
             declarationRefusals.delete(locator);
-            for (const attached of ['collection', 'typed-selection']) {
-                const key = modelKey(locator, attached);
-                models.delete(key);
-                liveModels.delete(key);
-                invalidModels.delete(key);
-            }
+            models.delete(locator);
+            liveModels.delete(locator);
+            invalidModels.delete(locator);
             journal.record(`room lost ${locator}`);
         }
         if (fresh.length === 0)
@@ -1258,7 +1278,14 @@ export function composeRuntime(options) {
         });
         await eraseFenced(fresh);
     }
-    function restored(key, persisted) {
+    function retyped(persisted) {
+        if (persisted.status === 'unavailable')
+            return persisted;
+        const { state } = attachedModel({ kind: 'model', snapshot: persisted.snapshot }, NO_MODEL, roomTypes);
+        return state.status === 'unavailable' ? state : { ...persisted, types: state.types };
+    }
+    function restored(key, held) {
+        const persisted = retyped(held);
         const prior = models.get(key);
         if (prior !== undefined &&
             prior.status !== 'unavailable' &&
@@ -1291,6 +1318,10 @@ export function composeRuntime(options) {
         transfers.clear();
         for (const [locator, record] of held.transfers)
             transfers.set(locator, record);
+        arrivals.clear();
+        keptParts.clear();
+        for (const [locator, parts] of held.kept)
+            keptParts.set(locator, new Map(parts));
         limits = held.providerLimits;
         capabilities = held.capabilities;
         member = held.member;
@@ -1316,9 +1347,7 @@ export function composeRuntime(options) {
         records.forEach((stored, index) => {
             if (fences.has(stored.locator))
                 return;
-            const model = restored(modelKey(stored.locator, 'collection'), stored.model);
-            const selectionModel = restored(modelKey(stored.locator, 'typed-selection'), stored.selectionModel);
-            const record = { ...stored, model, selectionModel };
+            const record = { ...stored, model: restored(stored.locator, stored.model) };
             replicas.set(record.locator, {
                 record,
                 queue: queuedBatches.get(record.locator) ?? [],
@@ -1336,13 +1365,10 @@ export function composeRuntime(options) {
                     roomVersion: record.roomVersion,
                     operations: record.operations,
                     replica: record.replica,
-                    model: modelTypes(model),
+                    model: modelTypes(record.model),
                     typedSelection: record.typedSelection === null
                         ? null
-                        : {
-                            roots: record.typedSelection.roots,
-                            model: modelTypes(selectionModel),
-                        },
+                        : { roots: record.typedSelection.roots, model: modelTypes(builtIn) },
                 });
             }
         });
@@ -1529,7 +1555,7 @@ export function composeRuntime(options) {
         return { count: answer.counts[locator] ?? replicas.get(locator)?.count ?? 0, indexed };
     }
     function replayed(record, confirmedLayer, queue) {
-        return stepped(replaying(confirmedLayer, queue, record.locator, compositionOf(record)));
+        return stepped(replaying(confirmedLayer, queue, record.locator, composed(record)));
     }
     async function overlayDiff(record, before, visible, touched) {
         const kept = before !== undefined && before.record.key === record.key
@@ -1592,7 +1618,7 @@ export function composeRuntime(options) {
         const confirmedLayer = overrides === undefined
             ? read
             : await stepped(slicingWith(read, overrides.set, overrides.removed));
-        const composition = compositionOf(record);
+        const composition = composed(record);
         const { withdrawn, kept } = await stepped(standing(confirmedLayer, queue, withdrawable(locator), localSaves, locator, composition));
         const founded = kept.filter((batch) => !queue.includes(batch));
         const queueChanged = kept.length !== previousQueue.length || founded.length > 0;
@@ -1698,35 +1724,25 @@ export function composeRuntime(options) {
         };
     }
     function adoptModels(locator, record) {
-        for (const [attached, model] of [
-            ['collection', record.model],
-            ['typed-selection', record.selectionModel],
-        ]) {
-            const key = modelKey(locator, attached);
-            models.set(key, model);
-            invalidModels.delete(key);
-            if (model.status === 'current')
-                liveModels.set(key, JSON.stringify(model.snapshot));
-            else if (model.status === 'unavailable')
-                liveModels.delete(key);
-        }
+        const model = record.model;
+        models.set(locator, model);
+        invalidModels.delete(locator);
+        if (model.status === 'current')
+            liveModels.set(locator, JSON.stringify(model.snapshot));
+        else if (model.status === 'unavailable')
+            liveModels.delete(locator);
     }
     function remodel(locator, answers, held) {
-        const taken = (answer, previous, attached) => {
-            if (answer === undefined)
-                return previous;
-            const { state, refused } = attachedModel(answer, previous, attached === 'collection' ? roomTypes : NO_ROOM_TYPES);
-            if (refused !== null) {
-                const where = refused.location === undefined ? '' : ` ${refused.location}`;
-                journal.record(`model invalid ${locator} ${attached} ${refused.identity} ${refused.rule}${where}`);
-            }
-            models.set(modelKey(locator, attached), state);
-            return state;
-        };
-        return {
-            collection: taken(answers?.collection, held.collection, 'collection'),
-            selection: taken(answers?.selection, held.selection, 'typed-selection'),
-        };
+        const answer = answers?.collection;
+        if (answer === undefined)
+            return held;
+        const { state, refused } = attachedModel(answer, held.collection, roomTypes);
+        if (refused !== null) {
+            const where = refused.location === undefined ? '' : ` ${refused.location}`;
+            journal.record(`model invalid ${locator} collection ${refused.identity} ${refused.rule}${where}`);
+        }
+        models.set(locator, state);
+        return { collection: state };
     }
     function publish(local, room, record, landing, live) {
         const landed = exclusive(async () => {
@@ -1753,14 +1769,14 @@ export function composeRuntime(options) {
                     ? {
                         room: true,
                         view: 'refreshed',
-                        commit: { refreshed: true, composition: compositionOf(record) },
+                        commit: { refreshed: true, composition: composed(record) },
                     }
                     : {
                         room: true,
                         view: landing.refreshed ? 'refreshed' : 'confirmed',
                         commit: {
                             published: { from: prior?.record.key ?? null },
-                            composition: compositionOf(record),
+                            composition: composed(record),
                             ...(landing.refreshed ? { refreshed: true } : {}),
                         },
                     });
@@ -1772,7 +1788,7 @@ export function composeRuntime(options) {
                 return true;
             }
             if (landing.kind === 'bootstrapped') {
-                transfers.delete(room.locator);
+                untransferred(room.locator);
                 catching.delete(room.locator);
                 sweep();
             }
@@ -1870,7 +1886,7 @@ export function composeRuntime(options) {
             : (await local.referrers(record.key, 'confirmed', removed, CHECKED_LINKS)).map(({ path }) => path);
         const confirmedLayer = await sliceAt(record, 'confirmed', replica.queue, [...paths, ...referring], moved);
         const { visible } = await replayed(record, confirmedLayer, replica.queue);
-        const composition = compositionOf(record);
+        const composition = composed(record);
         return {
             record,
             confirmed: confirmedLayer,
@@ -1886,13 +1902,7 @@ export function composeRuntime(options) {
             return false;
         const local = await openLocal();
         const roots = record.typedSelection?.roots ?? [];
-        if (composition.model.status === 'unavailable' &&
-            (await local.typedWithin(record.key, 'visible', roots, false))) {
-            return true;
-        }
-        return (composition.selection !== null &&
-            composition.selection.model.status === 'unavailable' &&
-            (await local.typedWithin(record.key, 'visible', roots, true)));
+        return local.typedWithin(record.key, 'visible', roots, false);
     }
     function textDiffers(left, right) {
         return ((left?.content ?? undefined) !== (right?.content ?? undefined) ||
@@ -1997,7 +2007,7 @@ export function composeRuntime(options) {
             const lookup = overriddenLookup(lookupOf(local, record.key, 'confirmed'), overrides);
             const clean = available(composition) &&
                 changed.length <= CLEAN_CHANGES &&
-                (await changedCleanly(changed, compositionOf(record), composition, {
+                (await changedCleanly(changed, composed(record), composition, {
                     lookup,
                     texts: textsOf(local, record.key, 'confirmed'),
                     referrers: referrersOf(local, record.key, 'confirmed'),
@@ -2754,8 +2764,8 @@ export function composeRuntime(options) {
         }
     }
     async function validatedRoom(local, record, view, previous, overrides = null) {
-        const composition = compositionOf(record);
-        const modeled = record.model.status !== 'unavailable' || record.selectionModel.status !== 'unavailable';
+        const composition = composed(record);
+        const modeled = record.model.status !== 'unavailable' || record.typedSelection !== null;
         const lookup = overrides === null
             ? lookupOf(local, record.key, view)
             : overriddenLookup(lookupOf(local, record.key, view), overrides);
@@ -2789,11 +2799,11 @@ export function composeRuntime(options) {
         }
         const named = changes === null ? null : changes.map(({ path }) => path);
         const textChanged = changes === null || changes.length > 0 || reshaped;
-        const modelsChanged = previous === null || !sameComposition(compositionOf(previous.record), composition);
+        const modelsChanged = previous === null || !sameComposition(composed(previous.record), composition);
         if (modeled && (textChanged || modelsChanged)) {
             const clean = previous !== null &&
                 changes !== null &&
-                (await changedCleanly(changes, compositionOf(previous.record), composition, {
+                (await changedCleanly(changes, composed(previous.record), composition, {
                     lookup,
                     texts,
                     referrers: referrersOf(local, record.key, 'confirmed'),
@@ -2817,7 +2827,7 @@ export function composeRuntime(options) {
             async turn() {
                 if (replaying === null)
                     return;
-                installing(replaying, 'replaying');
+                installing(replaying);
                 await screened();
             },
             prior: async (paths) => new Map((await local.entries(key, 'refreshed', paths)).map((entry) => [entry.path, entry])),
@@ -2831,7 +2841,7 @@ export function composeRuntime(options) {
                     const staged = await write(async () => {
                         if (!liveness())
                             return false;
-                        await timed(() => local.refresh(key, last
+                        await timed(stageBudget, () => local.refresh(key, last
                             ? { ...page, entries }
                             : { entries, removals: [], folders: [], folderRemovals: [] }));
                         return true;
@@ -2850,21 +2860,21 @@ export function composeRuntime(options) {
         let spent = 0;
         while (end < entries.length) {
             const size = (entries[end].content?.length ?? 0) + ROW_WEIGHT;
-            if (spent + size > stageBytes && end > from)
+            if (spent + size > stageBudget.bytes && end > from)
                 break;
             spent += size;
             end += 1;
         }
         return end;
     }
-    async function timed(work) {
+    async function timed(budget, work) {
         const started = performance.now();
         const done = await work();
         const spent = performance.now() - started;
-        if (spent < STAGE_AIM_MS / 2)
-            stageBytes = Math.min(STAGE_BYTES, Math.round(stageBytes * 1.5));
-        else if (spent > STAGE_AIM_MS * 2) {
-            stageBytes = Math.max(STAGE_FLOOR, Math.round(stageBytes / 1.5));
+        if (spent < budget.aim / 2)
+            budget.bytes = Math.min(budget.ceiling, Math.round(budget.bytes * 1.5));
+        else if (spent > budget.aim * 2) {
+            budget.bytes = Math.max(budget.floor, Math.round(budget.bytes / 1.5));
         }
         return done;
     }
@@ -2880,25 +2890,34 @@ export function composeRuntime(options) {
         return new Promise((resolve) => shownWaiters.push(resolve));
     }
     async function heldPart() {
-        while (partsFree === 0)
+        while (partsFree === 0 || queuedBytes >= QUEUED_BYTES) {
             await new Promise((admit) => partWaiters.push(admit));
+        }
         partsFree -= 1;
-        return freed();
+        return once(() => {
+            partsFree += 1;
+        });
     }
-    function freed() {
+    function once(release) {
         let released = false;
         return () => {
             if (released)
                 return;
             released = true;
-            partsFree += 1;
-            partWaiters.shift()?.();
+            release();
+            for (const admit of partWaiters.splice(0))
+                admit();
         };
     }
-    function installing(locator, step, share = null) {
-        installs.set(locator, { step, share });
+    function installing(locator, inventory = null) {
+        installs.set(locator, inventory ?? installs.get(locator) ?? null);
         queueReadiness();
     }
+    function untransferred(locator) {
+        transfers.delete(locator);
+        keptParts.delete(locator);
+        underived.delete(locator);
+    }
     function installed(locator) {
         if (installs.delete(locator))
             queueReadiness();
@@ -2965,11 +2984,34 @@ export function composeRuntime(options) {
         heldContents.set(key, made);
         return made;
     }
+    async function keptTogether({ local, parts }) {
+        const live = parts.filter((one) => one.live() && transfers.has(one.locator));
+        await local.keepParts(live.map(({ locator, token, part }) => ({ locator, token, part })));
+        for (const { locator, token, part } of live) {
+            const held = keptParts.get(locator) ?? new Map();
+            keptParts.set(locator, held.set(token, part.byteLength));
+            arrivals.get(locator)?.delete(token);
+        }
+        queueReadiness();
+    }
     function staging(local, liveness) {
+        const derived = async (locator, key) => {
+            underived.delete(locator);
+            for (;;) {
+                await screened();
+                if (!liveness())
+                    return false;
+                const done = await installWrite(async () => liveness() ? timed(deriveBudget, () => local.rederive(key, deriveBudget.bytes)) : null);
+                if (done === null)
+                    return false;
+                if (done)
+                    return true;
+            }
+        };
         return {
             held: (locator) => transfers.get(locator) ?? null,
             async turn(locator, step) {
-                installing(locator, step);
+                installing(locator);
                 for (;;) {
                     await screened();
                     const release = step === 'downloading' ? await heldPart() : () => { };
@@ -2978,17 +3020,17 @@ export function composeRuntime(options) {
                     release();
                 }
             },
-            spare() {
-                if (!onScreen || closed || partsFree === 0)
-                    return null;
-                partsFree -= 1;
-                return freed();
+            queued(bytes) {
+                queuedBytes += bytes;
+                return once(() => {
+                    queuedBytes -= bytes;
+                });
             },
             arriving(locator) {
                 let ended = false;
                 return {
                     at(token, bytes) {
-                        if (ended)
+                        if (ended || keptParts.get(locator)?.has(token) === true)
                             return;
                         const held = arrivals.get(locator) ?? new Map();
                         arrivals.set(locator, held.set(token, bytes));
@@ -3006,64 +3048,44 @@ export function composeRuntime(options) {
                 };
             },
             received: (locator, token) => local.transferPart(locator, token),
-            keep: (locator, token, received) => installWrite(async () => {
-                if (liveness())
-                    await local.keepPart(locator, token, received);
-            }),
+            keep: (locator, token, received) => gathered('kept parts', { local, parts: [{ locator, token, part: received, live: liveness }] }, (earlier, later) => earlier.local === later.local
+                ? { local: earlier.local, parts: [...earlier.parts, ...later.parts] }
+                : null, keptTogether),
             announced(locator) {
                 const later = announced.get(locator);
                 return later === undefined ? [] : [later];
             },
-            async stage(record, page, composition, received) {
+            async stage(record, page, composition, written) {
                 const { locator } = record;
-                const whole = received?.byteLength ?? 0;
-                const from = record.writing?.written ?? 0;
-                const token = record.writing?.token ?? null;
-                const entries = page.entries.length;
-                let held = record;
-                if (token !== null)
-                    arrivals.get(locator)?.delete(token);
-                installing(locator, 'writing', { written: from, entries, bytes: whole });
-                for (let written = from;;) {
-                    await screened();
+                installing(locator, record.inventory);
+                await screened();
+                if (!liveness())
+                    return null;
+                const key = await installWrite(async () => {
                     if (!liveness())
                         return null;
-                    const end = batchEnd(page.entries, written);
-                    const last = end === entries;
-                    const next = last
-                        ? {
-                            ...held,
-                            writing: null,
-                            done: {
-                                parts: held.done.parts + (received === null ? 0 : 1),
-                                bytes: held.done.bytes + whole,
-                            },
-                        }
-                        : { ...held, writing: { token, written: end } };
-                    const key = await installWrite(async () => {
-                        if (!liveness())
-                            return null;
-                        return timed(() => local.stage(next, composition, {
-                            entries: page.entries.slice(written, end),
-                            folders: last ? page.folders : [],
-                        }, last && received !== null && token !== null ? token : undefined));
-                    });
-                    if (key === null)
-                        return null;
-                    const arrived = !transfers.has(locator);
-                    held = { ...next, key };
-                    transfers.set(locator, held);
-                    if (arrived)
-                        liveQueries.recover();
-                    if (last) {
-                        installing(locator, held.pageToken === null ? 'replaying' : 'downloading');
-                        liveQueries.recover();
-                        return held;
-                    }
-                    installing(locator, 'writing', { written: end, entries, bytes: whole });
-                    written = end;
+                    return local.stage(record, composition, page, written ?? undefined);
+                });
+                if (key === null || !liveness())
+                    return null;
+                const arrived = !transfers.has(locator);
+                const held = { ...record, key };
+                transfers.set(locator, held);
+                if (written !== null) {
+                    keptParts.get(locator)?.delete(written);
+                    arrivals.get(locator)?.delete(written);
                 }
+                if (arrived)
+                    liveQueries.recover();
+                installing(locator);
+                liveQueries.recover();
+                const owed = page.entries.reduce((sum, entry) => sum + (entry.content?.length ?? 0), underived.get(locator) ?? 0);
+                underived.set(locator, owed);
+                if (owed < deriveBudget.bytes)
+                    return held;
+                return (await derived(locator, key)) ? held : null;
             },
+            derive: derived,
             async refreshing(locator, key) {
                 await exclusive(() => local.unrefresh(key));
                 return refreshingOf(local, key, liveness, locator);
@@ -3072,7 +3094,7 @@ export function composeRuntime(options) {
                 if (!liveness())
                     return;
                 await local.abandon(locator);
-                transfers.delete(locator);
+                untransferred(locator);
                 liveQueries.recover();
                 sweep();
             }),
@@ -3080,7 +3102,7 @@ export function composeRuntime(options) {
                 if (!liveness())
                     return;
                 await local.abandon(locator, true);
-                transfers.delete(locator);
+                untransferred(locator);
                 liveQueries.recover();
             }),
             release: (key) => exclusive(async () => {
@@ -3106,10 +3128,7 @@ export function composeRuntime(options) {
             liveQueries.recover();
         }
         try {
-            const result = await bootstrapRoom(reads, authorized, room, {
-                collection: modelOf(room.locator),
-                selection: modelOf(room.locator, 'typed-selection'),
-            }, (models, held) => (liveness() ? remodel(room.locator, models, held) : held), liveness, staging(local, liveness), reusable(room, local));
+            const result = await bootstrapRoom(reads, authorized, room, { collection: modelOf(room.locator) }, builtIn, (models, held) => (liveness() ? remodel(room.locator, models, held) : held), liveness, staging(local, liveness), reusable(room, local));
             const detail = result.kind === 'staged' ? result.record.roomVersion : result.kind;
             journal.record(`transfer bootstrap ${room.locator} ${detail}`);
             if (result.kind !== 'staged') {
@@ -3175,7 +3194,7 @@ export function composeRuntime(options) {
                         options.adapter.reached(room.locator, now, refreshed.record.roomVersion);
                     return reached ? null : advance(room, local, authorized, live);
                 }
-                await retainModel(local, room, 'collection', modelOf(room.locator));
+                await retainModel(local, room, modelOf(room.locator));
                 return null;
             }
             if (refreshed.kind !== 'beyond-horizon')
@@ -3771,9 +3790,9 @@ export function composeRuntime(options) {
             release();
         };
     }
-    async function retainModel(local, room, attached, next) {
-        const key = modelKey(room.locator, attached);
-        const previous = modelOf(room.locator, attached);
+    async function retainModel(local, room, next) {
+        const key = room.locator;
+        const previous = modelOf(room.locator);
         const heldBefore = replicas.get(room.locator)?.record;
         if (JSON.stringify(previous) === JSON.stringify(next) &&
             (heldBefore === undefined ||
@@ -3797,11 +3816,11 @@ export function composeRuntime(options) {
                     operations: room.operations,
                     replica: room.replica,
                     typedSelection: room.typedSelection ?? null,
-                    ...(attached === 'collection' ? { model: next } : { selectionModel: next }),
+                    model: next,
                 };
                 let read = 'staged';
                 if (!sameMembership(declared, held.record)) {
-                    const verdict = await validatedCollection(walkOf(local, held.record.key, 'confirmed'), lookupOf(local, held.record.key, 'confirmed'), compositionOf(declared));
+                    const verdict = await validatedCollection(walkOf(local, held.record.key, 'confirmed'), lookupOf(local, held.record.key, 'confirmed'), composed(declared));
                     read =
                         verdict.verdict !== 'refuse'
                             ? 'staged'
@@ -3822,7 +3841,7 @@ export function composeRuntime(options) {
                             key: record.key,
                             record: storedRecord(record),
                             roots: record.typedSelection?.roots ?? [],
-                            composition: compositionOf(record),
+                            composition: composed(record),
                             overlay: diff.overlay,
                         },
                     ],
@@ -3869,14 +3888,14 @@ export function composeRuntime(options) {
         });
         let scopedAway = room;
         if (result.scope === 'model')
-            await retainModel(local, room, 'collection', {
+            await retainModel(local, room, {
                 status: 'unavailable',
                 reason: 'not-readable',
             });
         else if (result.scope !== 'operation') {
             unreadable.add(room.locator);
             scopedAway = { ...room, replica: { ...room.replica, admission: 'unavailable' } };
-            await retainModel(local, scopedAway, 'collection', modelOf(room.locator));
+            await retainModel(local, scopedAway, modelOf(room.locator));
         }
         if (!continuing())
             return;
@@ -3926,7 +3945,7 @@ export function composeRuntime(options) {
             return false;
         }
         if (prior !== undefined)
-            await retainModel(local, room, 'collection', modelOf(room.locator));
+            await retainModel(local, room, modelOf(room.locator));
         if (discoveryPaused || authorization === undefined)
             return false;
         return room.replica.admission === 'admitted';
@@ -3939,10 +3958,8 @@ export function composeRuntime(options) {
             await withhold(withheld, room, local, progress);
     }
     function revoked(locator) {
-        return ['collection', 'typed-selection'].some((attached) => {
-            const model = modelOf(locator, attached);
-            return model.status === 'unavailable' && model.reason === 'revoked';
-        });
+        const model = modelOf(locator);
+        return model.status === 'unavailable' && model.reason === 'revoked';
     }
     async function loadRoom(room, local, progress, live) {
         if (!live() || authorization === undefined)
@@ -4000,10 +4017,7 @@ export function composeRuntime(options) {
         journal.record(`connection discover ${result.kind}`);
         if (result.kind === 'transport-uncertainty') {
             for (const room of rooms.values()) {
-                await retainModel(local, room, 'collection', cached(modelOf(room.locator)));
-                if (room.typedSelection !== null) {
-                    await retainModel(local, room, 'typed-selection', cached(modelOf(room.locator, 'typed-selection')));
-                }
+                await retainModel(local, room, cached(modelOf(room.locator)));
             }
             progress.reachable = false;
             progress.complete = false;
@@ -4285,12 +4299,9 @@ export function composeRuntime(options) {
         }
     }
     async function confirmModels(room, local) {
-        for (const attached of ['collection', 'typed-selection']) {
-            const model = modelOf(room.locator, attached);
-            if (model.status === 'cached') {
-                await retainModel(local, room, attached, { ...model, status: 'current' });
-            }
-        }
+        const model = modelOf(room.locator);
+        if (model.status === 'cached')
+            await retainModel(local, room, { ...model, status: 'current' });
     }
     function submitLater(locator) {
         if (localSaves)
@@ -4872,7 +4883,7 @@ export function composeRuntime(options) {
                 typedSelection: rooms.get(locator)?.typedSelection === undefined ||
                     rooms.get(locator)?.typedSelection === null
                     ? null
-                    : modelTypes(modelOf(locator, 'typed-selection')),
+                    : modelTypes(builtIn),
             };
         },
         roomDraft: {