@agentshouse/mdruntime 0.1.0-c1cc928d1d256 → 0.1.0-c2aaa9c0e3ca4

package/dist/connection.js

@@ -3,17 +3,22 @@ import { cached, modelTypes, NO_MODEL, sameModel, } from './model.js';
 import { observing } from './observation.js';
 import { filterOf, identityOf, planOf, } from './query.js';
 import { createReadSide } from './read-side.js';
+import { markdownPath, roomOfKey } from './rows.js';
 import { createSearchIndex, searchPlanOf, } from './search.js';
 import { bootstrapRoom } from './bootstrap.js';
 import { changeSet, mergeChangeSets } from './change-set.js';
 import { createConversationStore, } from './conversation.js';
-import { parseCollection, validRoot } from './collection.js';
+import { queuedFile } from './change.js';
+import { validRoot } from './collection.js';
+import { runtimeError } from './failure.js';
 import { createLog } from './log.js';
+import { binaryRecord } from './binary.js';
+import { absentBinary, binaryAnswer, calledResult, historyAnswer, undeclared, UNCERTAIN, } from './outcome.js';
 import { refreshRoom } from './refresh.js';
-import { buildBatch } from './save.js';
-import { applyReturned, compositionOf, landed, readConfirmed, readReclassified, replicaOf, sameMembership, withdrawals, } from './replica.js';
-import { changedParts, changesOf, held, readParts, returnedInto, underneath, } from './room-draft.js';
-import { settled, same, saving, STARTING, transferring, } from './status.js';
+import { buildBatch, joined, limitedIn, requeued } from './save.js';
+import { applyReturned, compositionOf, confirmedOf, landed, readReclassified, replicaOf, sameMembership, withdrawals, } from './replica.js';
+import { binaryBase, changedParts, changesOf, held, heldFile, joinedDraft, readParts, returnedBy, returnedInto, staleWrite, underneath, withdrawnDrafts, } from './room-draft.js';
+import { same, saving, STARTING, transferring, } from './status.js';
 import { detectStorageLoss, nextGeneration } from './durability.js';
 import { roomDraftRevision } from './identity.js';
 import { beneath, normalize } from './paths.js';
@@ -34,16 +39,19 @@ function declaredSelection(room) {
         selection.roots.every((other, at) => at === index || !within(other, root)));
 }
 export function composeRuntime(options) {
+    const localSaves = options.adapter.localSaves === true;
     let authorization = options.authorization;
     let opening;
     let store;
     const replicas = new Map();
+    const queuesWithoutReplica = new Map();
     const pendings = new Map();
     const roomDrafts = new Map();
     const presentations = new Map();
     let limits = [];
     let capabilities = [];
     const rooms = new Map();
+    const counted = new Map();
     const unreadable = new Set();
     const models = new Map();
     const liveModels = new Map();
@@ -82,6 +90,7 @@ export function composeRuntime(options) {
     let withdrawStream;
     const announced = new Map();
     let underway = null;
+    let streamed = null;
     let handed = false;
     const submitting = new Map();
     const answering = new Map();
@@ -107,9 +116,12 @@ export function composeRuntime(options) {
     const serviceModeListeners = new Set();
     let member = null;
     const memberListeners = new Set();
+    const materialized = new Set();
+    let everyRoom = false;
+    const transfers = new Map();
     const conversations = createConversationStore({
         agents: declaredAgents,
-        documents: (locator) => readSide.query(planOf({ rooms: [locator], select: 'all' }, roomFacts())),
+        documents: (locator) => rowsQuery(planOf({ rooms: [locator], select: 'all' }, roomFacts())),
         call: (name, input) => invokeAction(name, input),
         durably,
         stream: () => options.adapter.conversations,
@@ -185,7 +197,9 @@ export function composeRuntime(options) {
         }
     }
     function pending() {
-        return [...replicas.values()].some((replica) => replica.queue.length > 0);
+        return (!localSaves &&
+            (queuesWithoutReplica.size > 0 ||
+                [...replicas.values()].some((replica) => replica.queue.length > 0)));
     }
     function attention() {
         const attentionRequired = [];
@@ -214,8 +228,24 @@ export function composeRuntime(options) {
         }
         return { attentionRequired, reset };
     }
+    function documentsIn(locator) {
+        const replica = replicas.get(locator);
+        if (replica === undefined)
+            return 0;
+        const roots = replica.confirmed.record.typedSelection?.roots ?? [];
+        let count = 0;
+        for (const path of Object.keys(replica.visible.documents)) {
+            if (markdownPath(path) && !roots.some((root) => beneath(root, path)))
+                count += 1;
+        }
+        return count;
+    }
     function listed() {
-        return structuredClone([...rooms.values()]);
+        return structuredClone([...rooms.values()].map((room) => {
+            const documents = documentsIn(room.locator);
+            counted.set(room.locator, documents);
+            return { ...room, documents };
+        }));
     }
     function roomFacts(reach = null) {
         const held = [...rooms.values()]
@@ -270,6 +300,8 @@ export function composeRuntime(options) {
             listener(shown);
     }
     function entryOf(batch) {
+        if (localSaves)
+            return Object.freeze({ batch, state: 'local' });
         const state = pendings.get(batch.id);
         if (state?.accepted !== undefined) {
             return Object.freeze({ batch, state: 'accepted', roomVersion: state.accepted });
@@ -277,10 +309,7 @@ export function composeRuntime(options) {
         return Object.freeze({ batch, ...(state?.submission ?? { state: 'waiting' }) });
     }
     function entriesOf(locator) {
-        const replica = replicas.get(locator);
-        if (replica === undefined)
-            return [];
-        return replica.queue.map(entryOf);
+        return (replicas.get(locator)?.queue ?? queuesWithoutReplica.get(locator) ?? []).map(entryOf);
     }
     function emitQueue(locator) {
         const entries = entriesOf(locator);
@@ -290,12 +319,47 @@ export function composeRuntime(options) {
         }
     }
     function applyChangeSet(set) {
+        const held = everyRoom
+            ? set
+            : { ...set, documents: set.documents.filter(({ locator }) => materialized.has(locator)) };
+        readSide.apply(held);
+        searchIndex.changed(held);
+    }
+    function materialize(locators) {
+        if (everyRoom)
+            return;
+        if (locators === null)
+            everyRoom = true;
+        const sets = [];
+        for (const locator of locators ?? [...replicas.keys()]) {
+            if (materialized.has(locator))
+                continue;
+            materialized.add(locator);
+            const replica = replicas.get(locator);
+            if (replica !== undefined)
+                sets.push(changeSet(locator, null, replica));
+        }
+        if (sets.length === 0)
+            return;
+        const set = mergeChangeSets(sets);
         readSide.apply(set);
         searchIndex.changed(set);
     }
+    function reachedRooms(plan) {
+        if (plan.rooms !== null)
+            return plan.rooms;
+        if (plan.reach === null)
+            return null;
+        return [...plan.reach.rooms, ...plan.reach.documents.map(roomOfKey)];
+    }
+    function rowsQuery(plan) {
+        materialize(reachedRooms(plan));
+        return readSide.query(plan);
+    }
     function publishChangeSet(set) {
         applyChangeSet(set);
-        if (set.rooms.length > 0)
+        if (set.rooms.length > 0 ||
+            [...new Set(set.documents.map(({ locator }) => locator))].some((locator) => rooms.has(locator) && counted.get(locator) !== documentsIn(locator)))
             emitRooms();
         for (const locator of set.queues)
             emitQueue(locator);
@@ -342,10 +406,7 @@ export function composeRuntime(options) {
                     const revoked = { status: 'unavailable', reason: 'revoked' };
                     try {
                         const local = await openLocal();
-                        await local.revokeModels([...replicas.values()].map((replica) => ({
-                            record: replica.confirmed.record,
-                            queue: replica.queue,
-                        })));
+                        await local.revokeModels([...replicas.values()].map((replica) => replica.confirmed.record));
                     }
                     catch {
                         journal.record('model revocation persistence failed');
@@ -369,10 +430,7 @@ export function composeRuntime(options) {
                                 model: revoked,
                                 selectionModel: revoked,
                             };
-                            const read = readConfirmed(record, before.confirmed);
-                            replicas.set(locator, read.kind === 'staged'
-                                ? replicaOf(read.confirmed, before.queue, before)
-                                : { ...before, confirmed: { ...before.confirmed, record } });
+                            replicas.set(locator, replicaOf(confirmedOf(record, before.confirmed), before.queue, before));
                         }
                         rooms.set(locator, {
                             ...room,
@@ -389,7 +447,8 @@ export function composeRuntime(options) {
         }
         finally {
             refusal = {
-                reason,
+                code: reason,
+                retryable: false,
                 reauthorizationOffered: reason === 'revoked-authorization',
                 refusal: provider,
             };
@@ -414,7 +473,6 @@ export function composeRuntime(options) {
                 },
             providerOwnedPaths: room.providerOwnedPaths,
             roomVersion: replicas.get(room.locator)?.confirmed.record.roomVersion ?? null,
-            attachments: replicas.get(room.locator)?.confirmed.record.attachments ?? [],
             synchronizationAvailable,
             refusal: scopedRefusals.get(room.locator) ?? sourceRefusals.get(room.locator) ?? null,
         });
@@ -439,7 +497,14 @@ export function composeRuntime(options) {
         emitRooms();
     }
     function unlistedHeld(listed) {
-        return [...new Set([...rooms.keys(), ...roomDrafts.keys()])].filter((locator) => !listed.has(locator));
+        return [
+            ...new Set([
+                ...rooms.keys(),
+                ...roomDrafts.keys(),
+                ...transfers.keys(),
+                ...queuesWithoutReplica.keys(),
+            ]),
+        ].filter((locator) => !listed.has(locator));
     }
     async function dropUnlisted(listed, local) {
         if (unlistedHeld(listed).length === 0)
@@ -453,11 +518,13 @@ export function composeRuntime(options) {
                 drafts: [...(roomDrafts.get(locator)?.keys() ?? [])],
             }));
             for (const locator of unlisted) {
-                for (const batch of replicas.get(locator)?.queue ?? [])
+                for (const batch of replicas.get(locator)?.queue ?? queuesWithoutReplica.get(locator) ?? [])
                     pendings.delete(batch.id);
                 replicas.delete(locator);
+                queuesWithoutReplica.delete(locator);
                 rooms.delete(locator);
                 roomDrafts.delete(locator);
+                transfers.delete(locator);
                 unreadable.delete(locator);
                 stale.delete(locator);
                 scopedRefusals.delete(locator);
@@ -494,7 +561,7 @@ export function composeRuntime(options) {
         return model;
     }
     async function hydrate(opened) {
-        const [records, queues, kept, presented, held, chatDrafts, lastCompleted, declared, named, known,] = await Promise.all([
+        const [records, queues, kept, presented, held, chatDrafts, lastCompleted, declared, named, known, staged,] = await Promise.all([
             opened.all(),
             opened.queues(),
             opened.roomDrafts(),
@@ -505,7 +572,11 @@ export function composeRuntime(options) {
             opened.providerLimits(),
             opened.capabilities(),
             opened.member(),
+            opened.transfers(),
         ]);
+        transfers.clear();
+        for (const record of staged)
+            transfers.set(record.locator, record);
         limits = declared;
         capabilities = named;
         member = known;
@@ -524,18 +595,17 @@ export function composeRuntime(options) {
         }
         const previousReplicas = new Map(replicas);
         replicas.clear();
+        queuesWithoutReplica.clear();
+        const replicated = new Set(records.map((record) => record.locator));
+        for (const [locator, queue] of queuedBatches) {
+            if (!replicated.has(locator))
+                queuesWithoutReplica.set(locator, queue);
+        }
         for (const record of records) {
             const model = restored(modelKey(record.locator, 'collection'), record.model);
             const selectionModel = restored(modelKey(record.locator, 'typed-selection'), record.selectionModel);
             const prior = previousReplicas.get(record.locator);
-            const retyped = { ...record, model, selectionModel };
-            const read = readConfirmed(retyped, prior?.confirmed);
-            const confirmed = read.kind === 'staged'
-                ? read.confirmed
-                : {
-                    record: retyped,
-                    collection: parseCollection(retyped.documents, compositionOf(retyped)),
-                };
+            const confirmed = confirmedOf({ ...record, model, selectionModel }, prior?.confirmed);
             replicas.set(record.locator, replicaOf(confirmed, queuedBatches.get(record.locator) ?? [], prior));
             const existing = rooms.get(record.locator);
             if (existing === undefined)
@@ -544,7 +614,6 @@ export function composeRuntime(options) {
                 rooms.set(record.locator, {
                     ...existing,
                     roomVersion: record.roomVersion,
-                    attachments: record.attachments,
                     operations: record.operations,
                     replica: record.replica,
                     model: modelTypes(model),
@@ -556,10 +625,6 @@ export function composeRuntime(options) {
                         },
                 });
             }
-            if (read.kind === 'refused') {
-                refusedSource(record, read);
-                declarationRefusals.add(record.locator);
-            }
         }
         return {
             lastCompleted,
@@ -621,11 +686,12 @@ export function composeRuntime(options) {
         return state !== undefined && state.submission.state !== 'submitted' && !state.lost;
     }
     async function settle(local, locator, before, confirmed, queue, settling = {}) {
-        const withdrawn = withdrawals(confirmed, queue, withdrawable);
+        const previousQueue = before?.queue ?? queuesWithoutReplica.get(locator) ?? [];
+        const withdrawn = withdrawals(confirmed, queue, withdrawable, localSaves);
         const kept = queue.filter((batch) => !withdrawn.includes(batch));
         const unqueued = withdrawn.map((batch) => ({
             batch,
-            returned: landed(batch, confirmed) ? null : { kind: 'withdrawn' },
+            returned: localSaves || landed(batch, confirmed) ? null : returnedBy('withdrawn'),
         }));
         const returning = [
             ...[...unqueued].reverse(),
@@ -646,26 +712,44 @@ export function composeRuntime(options) {
                 });
             }
         }
-        await local.write({
-            next: confirmed.record,
-            previous: before?.confirmed.record ?? null,
-            queue: kept,
-            previousQueue: before?.queue ?? [],
-            roomDrafts: [...written.values()],
-        });
-        for (const batch of before?.queue ?? []) {
+        await local.keepAuthored({ locator, queue: kept, roomDrafts: [...written.values()] });
+        for (const batch of previousQueue) {
             if (!kept.includes(batch))
                 pendings.delete(batch.id);
         }
         for (const record of written.values())
             rememberRoomDraft(record);
+        try {
+            await local.write({
+                next: confirmed.record,
+                previous: before?.confirmed.record ?? null,
+                ...(settling.stored === undefined ? {} : { stored: settling.stored }),
+            });
+        }
+        catch (error) {
+            const held = before === undefined ? null : replicaOf(before.confirmed, kept, before);
+            if (held !== null)
+                replicas.set(locator, held);
+            else if (kept.length > 0)
+                queuesWithoutReplica.set(locator, kept);
+            else
+                queuesWithoutReplica.delete(locator);
+            publishChangeSet(changeSet(locator, before ?? null, held, {
+                queue: kept.length !== previousQueue.length,
+                drafts: [...written.keys()],
+            }));
+            moveTo({ ...status, ...attention() });
+            throw error;
+        }
+        queuesWithoutReplica.delete(locator);
         for (const { batch, returned } of unqueued) {
-            journal.record(`outcome ${locator} ${batch.id} ${returned === null ? 'landed' : 'withdrawn'}`);
+            const outcome = localSaves ? 'dropped' : returned === null ? 'landed' : 'withdrawn';
+            journal.record(`outcome ${locator} ${batch.id} ${outcome}`);
         }
         replicas.set(locator, replicaOf(confirmed, kept, before));
         return changeSet(locator, before ?? null, replicas.get(locator) ?? null, {
             room: settling.room,
-            queue: kept.length !== (before?.queue.length ?? 0),
+            queue: kept.length !== previousQueue.length,
             drafts: [...written.keys()],
         });
     }
@@ -700,7 +784,8 @@ export function composeRuntime(options) {
             selection: taken(answers?.selection, held.selection, 'typed-selection'),
         };
     }
-    function publish(local, room, confirmed, base) {
+    function publish(local, room, confirmed, landing = {}) {
+        const { base, stored } = landing;
         return exclusive(async () => {
             if (!continuing())
                 return true;
@@ -711,13 +796,15 @@ export function composeRuntime(options) {
                 return false;
             }
             adoptModels(room.locator, confirmed.record);
-            const queue = prior?.queue ?? [];
+            const queue = prior?.queue ?? queuesWithoutReplica.get(room.locator) ?? [];
             const roomVersion = confirmed.record.roomVersion;
             const accepted = queue.filter((batch) => {
                 const accepted = pendings.get(batch.id)?.accepted;
                 return (accepted !== undefined && options.adapter.reached(room.locator, roomVersion, accepted));
             });
-            const set = await settle(local, room.locator, prior, confirmed, queue.filter((batch) => !accepted.includes(batch)), { room: true });
+            const set = await settle(local, room.locator, prior, confirmed, queue.filter((batch) => !accepted.includes(batch)), { room: true, ...(stored === undefined ? {} : { stored }) });
+            if (stored !== undefined)
+                transfers.delete(room.locator);
             for (const batch of accepted)
                 pendings.delete(batch.id);
             stale.delete(room.locator);
@@ -736,22 +823,40 @@ export function composeRuntime(options) {
         const replica = replicas.get(locator);
         const room = rooms.get(locator);
         if (replica === undefined || room === undefined) {
-            throw new Error(`No Local Replica for ${locator}`);
+            throw runtimeError('room-not-held', `No Local Replica for ${locator}`);
         }
         const saved = [];
         for (const { path, revision } of named) {
             const record = roomDrafts.get(locator)?.get(path);
             if (record === undefined || record.revision !== revision) {
-                return { outcome: 'refused', rule: 'stale', file: path };
+                return { outcome: 'refused', code: 'stale', retryable: false, file: path };
             }
             if (!saved.some((draft) => draft.path === path))
                 saved.push({ path, parts: record.parts });
         }
         const changed = saved.flatMap(({ path, parts }) => changedParts(path, underneath(parts, path, replica.visible)));
         if (changed.length > 0)
-            return { outcome: 'refused', rule: 'changed-underneath', parts: changed };
+            return { outcome: 'refused', code: 'changed-underneath', retryable: false, parts: changed };
         const parts = saved.map(({ path, parts }) => changesOf(path, parts));
-        const built = buildBatch({
+        const changes = parts.flatMap(({ changes }) => changes);
+        const requeue = requeued(replica.queue, changes, replica.visible, withdrawable, (queue) => replicaOf(replica.confirmed, queue, replica).visible);
+        if (requeue.kind === 'refused')
+            return { outcome: 'refused', ...requeue.refusal };
+        const foreign = [...requeue.withdrawn.keys()].some((origin) => {
+            const writer = replica.queue.find((batch) => batch.id === origin)?.writtenBy;
+            return writer?.app !== writtenBy?.app || writer?.release !== writtenBy?.release;
+        });
+        if (foreign) {
+            return {
+                outcome: 'refused',
+                code: 'unknown-revision',
+                retryable: false,
+                file: [...requeue.queued].at(-1),
+            };
+        }
+        const withdrawing = requeue.withdrawn.size > 0;
+        const input = {
+            local: localSaves,
             baseRoomVersion: replica.confirmed.record.roomVersion,
             composition: compositionFor(replica.confirmed.record),
             operations: room.operations,
@@ -759,29 +864,86 @@ export function composeRuntime(options) {
             providerLimits: limits,
             handle: room.handle,
             visible: replica.visible,
-            changes: parts.flatMap(({ changes }) => changes),
+            changes,
             bases: parts.flatMap(({ bases }) => bases),
             writtenBy,
+        };
+        const checked = withdrawing ? buildBatch({ ...input, queued: requeue.queued }) : null;
+        if (checked?.kind === 'refused')
+            return { outcome: 'refused', ...checked.refusal };
+        const owners = [...requeue.owners].flatMap((origin) => {
+            const batch = requeue.withdrawn.get(origin);
+            return batch === undefined || batch === null ? [] : [{ origin, batch }];
         });
-        if (built.kind === 'refused')
+        if (owners.length > 1) {
+            return {
+                outcome: 'refused',
+                code: 'unknown-revision',
+                retryable: false,
+                file: [...requeue.queued].at(-1),
+            };
+        }
+        const built = !withdrawing
+            ? buildBatch(input)
+            : requeue.changes.length === 0
+                ? null
+                : buildBatch({
+                    ...input,
+                    visible: replicaOf(replica.confirmed, requeue.queue, replica).visible,
+                    changes: requeue.changes,
+                });
+        if (built?.kind === 'refused')
             return { outcome: 'refused', ...built.refusal };
-        const batches = [...replica.queue, built.batch];
+        const [owner] = owners;
+        const join = built === null || owner === undefined
+            ? null
+            : joined(requeue.queue, owner.batch, built.batch);
+        if (join?.kind === 'refused')
+            return { outcome: 'refused', ...join.refusal };
+        const withdrawn = new Map(requeue.withdrawn);
+        if (join !== null && owner !== undefined)
+            withdrawn.set(owner.origin, join.batch);
+        const drafted = new Map();
+        for (const [origin, replaced] of withdrawn) {
+            if (replaced === null)
+                continue;
+            const carried = withdrawnDrafts(pendings.get(origin)?.drafts ?? [], changes);
+            drafted.set(replaced.id, replaced === join?.batch
+                ? [
+                    ...carried.map((held) => joinedDraft(held, saved)),
+                    ...saved.filter((draft) => !carried.some((held) => held.path === draft.path)),
+                ]
+                : carried);
+        }
+        if (built !== null && join === null)
+            drafted.set(built.batch.id, saved);
+        const batches = join !== null ? join.queue : built === null ? requeue.queue : [...requeue.queue, built.batch];
+        for (const replaced of withdrawn.values()) {
+            if (replaced === null || !batches.includes(replaced))
+                continue;
+            const limited = limitedIn(batches, replaced, limits, (queue) => replicaOf(replica.confirmed, queue, replica).visible);
+            if (limited !== null)
+                return { outcome: 'refused', ...limited };
+        }
         const consumed = saved.map(({ path }) => path);
         try {
-            await local.write({
-                next: replica.confirmed.record,
-                previous: replica.confirmed.record,
+            await local.keepAuthored({
+                locator,
                 queue: batches,
-                previousQueue: replica.queue,
-                saved,
+                saved: drafted,
                 consumedRoomDrafts: consumed,
             });
         }
         catch {
-            return { outcome: 'refused', rule: 'durable-write' };
+            return { outcome: 'refused', code: 'durable-write', retryable: true };
         }
-        pendings.set(built.batch.id, { drafts: saved, submission: { state: 'waiting' }, lost: false });
-        recorded = false;
+        for (const origin of withdrawn.keys())
+            pendings.delete(origin);
+        for (const [id, drafts] of drafted) {
+            pendings.set(id, { drafts, submission: { state: 'waiting' }, lost: false });
+        }
+        if (!localSaves)
+            recorded = false;
         for (const path of consumed)
             forgetRoomDraft(locator, path);
         replicas.set(locator, replicaOf(replica.confirmed, batches, replica));
@@ -789,9 +951,12 @@ export function composeRuntime(options) {
             queue: true,
             drafts: consumed,
         }));
-        moveTo({ ...saving(status), ...attention() });
+        moveTo({ ...(localSaves ? status : saving(status)), ...attention() });
         submitLater(locator);
-        return { outcome: 'saved', identifier: built.batch.id };
+        const identifier = join?.batch.id ??
+            built?.batch.id ??
+            [...requeue.withdrawn].map(([origin, replaced]) => replaced?.id ?? origin).at(-1);
+        return { outcome: 'saved', identifier };
     }
     async function mark(local, locator, identifier, submission, lost) {
         const state = pendings.get(identifier);
@@ -817,16 +982,15 @@ export function composeRuntime(options) {
     }
     async function readHistory(locator, path, query = {}) {
         if (closed || refusal !== null)
-            return { kind: 'transport-uncertainty' };
+            return UNCERTAIN;
         try {
             await ensureAuthorization();
         }
         catch {
-            return { kind: 'transport-uncertainty' };
-        }
-        if (closed || authorization === undefined || refusal !== null) {
-            return { kind: 'transport-uncertainty' };
+            return UNCERTAIN;
         }
+        if (closed || authorization === undefined || refusal !== null)
+            return UNCERTAIN;
         const transfer = ['transfer', 'history', locator, path];
         if (query.revision !== undefined)
             transfer.push(query.revision);
@@ -848,12 +1012,96 @@ export function composeRuntime(options) {
             }
             if (result.kind === 'revoked-authorization')
                 await refuse('revoked-authorization');
-            return result;
+            return historyAnswer(result);
         }
         catch {
             journal.record('outcome history transport-uncertainty');
-            return { kind: 'transport-uncertainty' };
+            return UNCERTAIN;
+        }
+    }
+    function queuedBlob(locator, path, localId) {
+        for (const batch of replicas.get(locator)?.queue ?? []) {
+            const change = batch.changes[queuedFile(batch, path, localId)];
+            if (change?.kind === 'binary')
+                return change.blob;
         }
+        return null;
+    }
+    async function readBinary(request) {
+        if (closed || refusal !== null)
+            return UNCERTAIN;
+        try {
+            await ensureAuthorization();
+        }
+        catch {
+            return UNCERTAIN;
+        }
+        if (closed || authorization === undefined || refusal !== null)
+            return UNCERTAIN;
+        journal.record(request.kind === 'path'
+            ? `transfer binary ${request.locator} ${request.path}`
+            : `transfer binary ${request.fileRef}`);
+        try {
+            const result = await options.adapter.binary(authorization, request);
+            journal.record(`outcome binary ${result.kind}`);
+            if (result.kind === 'scoped-refusal') {
+                const room = replicas.get(result.locator)?.confirmed.record;
+                if (room !== undefined)
+                    await scoped(result, room, await openLocal());
+            }
+            if (result.kind === 'revoked-authorization')
+                await refuse('revoked-authorization');
+            return binaryAnswer(result);
+        }
+        catch {
+            journal.record('outcome binary transport-uncertainty');
+            return UNCERTAIN;
+        }
+    }
+    async function readBinaryPath(locator, path, revision) {
+        await ready;
+        if (revision !== undefined)
+            return readBinary({ kind: 'path', locator, path, revision });
+        const visible = replicas.get(locator)?.visible;
+        const held = visible?.binaries[path];
+        if (visible === undefined || held === undefined)
+            return absentBinary(path);
+        if (held.state === 'pending') {
+            const blob = queuedBlob(locator, path, held.localId);
+            return blob === null ? absentBinary(path) : { kind: 'binary', blob };
+        }
+        const record = binaryRecord(path, held, visible.revisions[path]);
+        if (record.state !== 'accepted')
+            return absentBinary(path);
+        const replica = replicas.get(locator);
+        const queued = (replica?.queue ?? []).flatMap((batch) => batch.changes);
+        const holds = (at) => {
+            const kept = replica?.confirmed.record.binaries[at];
+            return (kept?.state === 'accepted' &&
+                kept.fileRef === record.fileRef &&
+                replica?.confirmed.record.revisions[at] === record.revision);
+        };
+        let confirmed = path;
+        for (let index = queued.length - 1; index >= 0 && !holds(confirmed); index -= 1) {
+            const change = queued[index];
+            if (change.kind === 'move' && change.to === confirmed)
+                confirmed = change.path;
+        }
+        return readBinary({
+            kind: 'path',
+            locator,
+            path: confirmed,
+            revision: record.revision,
+            fileRef: record.fileRef,
+        });
+    }
+    async function readBinaryReference(reference) {
+        await ready;
+        return readBinary({
+            kind: 'reference',
+            fileRef: reference.fileRef,
+            referrer: reference.referrer,
+        });
     }
     function declaredAgents() {
         for (const capability of capabilities) {
@@ -874,18 +1122,17 @@ export function composeRuntime(options) {
     async function invokeAction(name, input) {
         await ready;
         if (closed || refusal !== null)
-            return { kind: 'transport-uncertainty' };
+            return UNCERTAIN;
         if (declaredAction(name) === undefined)
-            return { kind: 'undeclared' };
+            return undeclared(name);
         try {
             await ensureAuthorization();
         }
         catch {
-            return { kind: 'transport-uncertainty' };
-        }
-        if (closed || authorization === undefined || refusal !== null) {
-            return { kind: 'transport-uncertainty' };
+            return UNCERTAIN;
         }
+        if (closed || authorization === undefined || refusal !== null)
+            return UNCERTAIN;
         journal.record(`transfer action ${name}`);
         try {
             const result = await options.adapter.action(authorization, { name, input });
@@ -899,11 +1146,11 @@ export function composeRuntime(options) {
             journal.record(`outcome action ${result.kind}`);
             if (result.kind === 'rejected')
                 journal.record(`refusal rejected ${result.code}`);
-            return result;
+            return calledResult(result);
         }
         catch {
             journal.record('outcome action transport-uncertainty');
-            return { kind: 'transport-uncertainty' };
+            return UNCERTAIN;
         }
     }
     function continuing() {
@@ -934,14 +1181,33 @@ export function composeRuntime(options) {
             throw error;
         }
     }
-    async function pullBootstrap(authorized, room) {
+    function staging(local) {
+        return {
+            held: (locator) => transfers.get(locator) ?? null,
+            staged: (record) => local.staged(record),
+            stage: (record, page) => exclusive(async () => {
+                if (!continuing())
+                    return false;
+                await local.stage(record, page);
+                transfers.set(record.locator, record);
+                return true;
+            }),
+            abandon: (locator) => exclusive(async () => {
+                if (!continuing())
+                    return;
+                await local.abandon(locator);
+                transfers.delete(locator);
+            }),
+        };
+    }
+    async function pullBootstrap(authorized, room, local) {
         if (!continuing())
             return unconfirmed;
         try {
             const result = await bootstrapRoom(options.adapter, authorized, room, {
                 collection: modelOf(room.locator),
                 selection: modelOf(room.locator, 'typed-selection'),
-            }, (models, held) => remodel(room.locator, models, held), continuing);
+            }, (models, held) => remodel(room.locator, models, held), continuing, staging(local));
             const detail = result.kind === 'staged' ? result.confirmed.record.roomVersion : result.kind;
             journal.record(`transfer bootstrap ${room.locator} ${detail}`);
             return result;
@@ -969,7 +1235,7 @@ export function composeRuntime(options) {
                 return null;
             }
             if (refreshed.kind === 'advanced') {
-                if (!(await publish(local, room, refreshed.confirmed, held.confirmed))) {
+                if (!(await publish(local, room, refreshed.confirmed, { base: held.confirmed }))) {
                     const now = replicas.get(room.locator)?.confirmed.record.roomVersion;
                     const reached = now !== undefined &&
                         options.adapter.reached(room.locator, now, refreshed.confirmed.record.roomVersion);
@@ -981,12 +1247,12 @@ export function composeRuntime(options) {
             if (refreshed.kind !== 'beyond-horizon')
                 return refreshed;
         }
-        const bootstrapped = await pullBootstrap(authorized, room);
+        const bootstrapped = await pullBootstrap(authorized, room, local);
         if (!continuing())
             return unconfirmed;
         if (bootstrapped.kind !== 'staged')
             return bootstrapped;
-        await publish(local, room, bootstrapped.confirmed);
+        await publish(local, room, bootstrapped.confirmed, { stored: bootstrapped.stored });
         return null;
     }
     async function withhold(withheld, room, local, progress) {
@@ -1018,6 +1284,7 @@ export function composeRuntime(options) {
     function refusedSource(room, { refusal }) {
         sourceRefusals.set(room.locator, {
             code: refusal.rule,
+            retryable: false,
             fields: {
                 file: refusal.file,
                 ...(refusal.location === undefined ? {} : { location: refusal.location }),
@@ -1032,8 +1299,16 @@ export function composeRuntime(options) {
     }
     function leaving(batch, result) {
         if (result.kind === 'expired')
-            return { batch, returned: { kind: 'expired' } };
-        return { batch, returned: { kind: result.kind, code: result.code, fields: result.fields } };
+            return { batch, returned: returnedBy('expired') };
+        return {
+            batch,
+            returned: {
+                kind: result.kind,
+                code: result.code,
+                fields: result.fields,
+                retryable: result.retryable,
+            },
+        };
     }
     function logAccepted(locator, submitted, accepted) {
         journal.record(`outcome ${locator} ${submitted.id} accepted ${accepted.roomVersion}` +
@@ -1051,7 +1326,7 @@ export function composeRuntime(options) {
         logAccepted(room.locator, submitted, accepted);
         const applied = options.adapter.reached(room.locator, held, accepted.roomVersion)
             ? { kind: 'staged', confirmed: current.confirmed }
-            : applyReturned(current.confirmed, accepted.documents, accepted.revisions, accepted.removals, accepted.attachments, options.adapter.follows(room.locator, held, accepted.roomVersion)
+            : applyReturned(current.confirmed, submitted, accepted, options.adapter.follows(room.locator, held, accepted.roomVersion)
                 ? accepted.roomVersion
                 : held, accepted.roomVersion);
         if (applied.kind !== 'staged') {
@@ -1259,7 +1534,7 @@ export function composeRuntime(options) {
                     ...(attached === 'collection' ? { model: next } : { selectionModel: next }),
                 };
                 const reclassified = sameMembership(declared, held.confirmed.record)
-                    ? readConfirmed(declared, held.confirmed)
+                    ? { kind: 'staged', confirmed: confirmedOf(declared, held.confirmed) }
                     : readReclassified(declared, held.confirmed);
                 const record = reclassified.kind === 'staged'
                     ? declared
@@ -1267,12 +1542,10 @@ export function composeRuntime(options) {
                         ...declared,
                         typedSelection: held.confirmed.record.typedSelection,
                     };
-                const read = reclassified.kind === 'deferred' ? readConfirmed(record, held.confirmed) : reclassified;
-                await local.write({
-                    next: record,
-                    previous: held.confirmed.record,
-                    queue: held.queue,
-                });
+                const read = reclassified.kind === 'deferred'
+                    ? { kind: 'staged', confirmed: confirmedOf(record, held.confirmed) }
+                    : reclassified;
+                await local.write({ next: record, previous: held.confirmed.record });
                 if (read.kind === 'staged') {
                     replicas.set(room.locator, replicaOf(read.confirmed, held.queue, held));
                     if (declarationRefusals.delete(room.locator)) {
@@ -1299,7 +1572,11 @@ export function composeRuntime(options) {
         });
     }
     async function scoped(result, room, local) {
-        scopedRefusals.set(room.locator, { code: result.code, fields: result.fields });
+        scopedRefusals.set(room.locator, {
+            code: result.code,
+            fields: result.fields,
+            retryable: result.retryable,
+        });
         let scopedAway = room;
         if (result.scope === 'model')
             await retainModel(local, room, 'collection', {
@@ -1336,7 +1613,6 @@ export function composeRuntime(options) {
                     roomVersion: record.roomVersion,
                     documents: record.documents,
                     revisions: record.revisions,
-                    attachments: record.attachments,
                 },
             ];
         });
@@ -1416,14 +1692,14 @@ export function composeRuntime(options) {
     async function reload(room, local, progress) {
         if (discoveryPaused || authorization === undefined)
             return;
-        const bootstrapped = await pullBootstrap(authorization, room);
+        const bootstrapped = await pullBootstrap(authorization, room, local);
         if (!continuing())
             return;
         if (bootstrapped.kind !== 'staged') {
             await withhold(bootstrapped, room, local, progress);
             return;
         }
-        await publish(local, room, bootstrapped.confirmed);
+        await publish(local, room, bootstrapped.confirmed, { stored: bootstrapped.stored });
         await caughtUp(room, local, progress);
     }
     async function discoverRooms(local, progress, attempt) {
@@ -1458,7 +1734,11 @@ export function composeRuntime(options) {
         }
         if (result.kind === 'contract-unavailable') {
             journal.record(`connection contract unavailable ${result.code}`);
-            await refuse('missing-required-guarantee', { code: result.code, fields: result.fields });
+            await refuse('missing-required-guarantee', {
+                code: result.code,
+                fields: result.fields,
+                retryable: result.retryable,
+            });
             notifyReady();
             return;
         }
@@ -1534,6 +1814,8 @@ export function composeRuntime(options) {
             }
             const progress = { complete: true, reachable: true };
             underway = attempt === 'opened' ? progress : null;
+            if (attempt === 'opened')
+                streamed = progress;
             complete = false;
             recorded = false;
             moveTo(transferring(status));
@@ -1574,9 +1856,14 @@ export function composeRuntime(options) {
             void refuse('revoked-authorization');
             return;
         }
-        if (opened || status.providerUnreachable)
-            return;
-        moveTo({ ...status, providerUnreachable: true });
+        complete = false;
+        if (streamed !== null)
+            streamed.complete = false;
+        moveTo({
+            ...status,
+            state: status.state === 'synced' ? 'syncing' : status.state,
+            providerUnreachable: status.providerUnreachable || !opened,
+        });
     }
     function applyOpening(expired) {
         return task(async () => {
@@ -1631,6 +1918,8 @@ export function composeRuntime(options) {
         }
     }
     function submitLater(locator) {
+        if (localSaves)
+            return;
         const running = submitting.get(locator);
         if (running !== undefined) {
             running.again = true;
@@ -1710,7 +1999,7 @@ export function composeRuntime(options) {
             }
         }
         moveTo({
-            state: pending() ? 'pending' : synced ? 'synced' : 'syncing',
+            state: pending() ? 'pending' : synced && complete ? 'synced' : 'syncing',
             lastCompleted,
             providerUnreachable: !progress.reachable,
             ...attention(),
@@ -1819,20 +2108,15 @@ export function composeRuntime(options) {
             show();
             return;
         }
-        const resting = status;
-        const progress = { complete: false, reachable: !resting.providerUnreachable };
-        moveTo(transferring(resting));
+        const progress = { complete: false, reachable: !status.providerUnreachable };
+        moveTo(transferring(status));
         try {
             await conversations.catchUp(progress);
         }
         catch {
             progress.reachable = false;
         }
-        finally {
-            if (!closed) {
-                moveTo(settled(resting, progress, resting.lastCompleted, pending(), attention()));
-            }
-        }
+        await rest(progress);
     }
     function close() {
         if (closed)
@@ -1886,13 +2170,16 @@ export function composeRuntime(options) {
         return planOf(input, roomFacts(reach), planReach(reach));
     }
     function watchPlan(plan, deliver) {
+        materialize(reachedRooms(plan));
         return readSide.watch(plan, identityOf(plan), deliver);
     }
     async function searched(input, reach) {
         const plan = searchPlanOf(input);
         const selected = plan.documents === undefined && reach === null
             ? null
-            : (await readSide.query(filterOf(plan.documents ?? {}, roomFacts(reach), planReach(reach)))).documents;
+            : (await rowsQuery(filterOf(plan.documents ?? {}, roomFacts(reach), planReach(reach)))).documents;
+        if (selected === null)
+            materialize(null);
         return searchIndex.search(reach === null || reach.conversations ? plan : { ...plan, conversations: [] }, selected);
     }
     const watch = (topic, input, deliver) => {
@@ -1918,6 +2205,8 @@ export function composeRuntime(options) {
         }
         if (topic === 'conversations')
             return conversations.watchList(deliver);
+        if (topic === 'agents')
+            return conversations.watchAgents(deliver);
         if (topic === 'presentation') {
             const listener = {
                 app: input[0],
@@ -1996,7 +2285,7 @@ export function composeRuntime(options) {
         rooms() {
             return listed();
         },
-        query: ((input) => readSide.query(planned(input, null))),
+        query: ((input) => rowsQuery(planned(input, null))),
         search: (input) => searched(input, null),
         types(locator) {
             return {
@@ -2025,20 +2314,20 @@ export function composeRuntime(options) {
                 return exclusive(async () => {
                     const parts = readParts(input);
                     if (path === '' || normalize(path) !== path)
-                        throw new Error('room-draft-malformed');
+                        throw runtimeError('room-draft-malformed');
                     const local = await openLocal();
                     const current = roomDrafts.get(room)?.get(path);
                     if ((current?.revision ?? null) !== revision) {
-                        return {
-                            outcome: 'stale',
-                            current: current === undefined ? null : roomDraftOf(current),
-                        };
+                        return staleWrite(current === undefined ? null : roomDraftOf(current));
                     }
+                    const file = parts.file === undefined
+                        ? undefined
+                        : await heldFile(parts.file.blob, binaryBase(replicas.get(room)?.visible, path));
                     const record = {
                         locator: room,
                         path,
                         revision: roomDraftRevision(),
-                        parts: held(current?.parts, parts),
+                        parts: held(current?.parts, parts, file),
                         returned: current?.returned ?? null,
                     };
                     await local.keepRoomDraft(record);
@@ -2052,10 +2341,7 @@ export function composeRuntime(options) {
                     const local = await openLocal();
                     const current = roomDrafts.get(room)?.get(path);
                     if (current === undefined || current.revision !== revision) {
-                        return {
-                            outcome: 'stale',
-                            current: current === undefined ? null : roomDraftOf(current),
-                        };
+                        return staleWrite(current === undefined ? null : roomDraftOf(current));
                     }
                     await local.discardRoomDraft(room, path);
                     forgetRoomDraft(room, path);
@@ -2090,6 +2376,10 @@ export function composeRuntime(options) {
         history(locator, path, query) {
             return readHistory(locator, path, query);
         },
+        binaryFile: {
+            read: readBinaryPath,
+            readReference: readBinaryReference,
+        },
         call(name, input) {
             return invokeAction(name, input);
         },
@@ -2122,8 +2412,9 @@ export function composeRuntime(options) {
     const reachable = {
         md,
         watch,
+        referrerScope: (referrer) => options.adapter.referrerScope(referrer),
         plan: planned,
-        query: (plan) => readSide.query(plan),
+        query: (plan) => rowsQuery(plan),
         watchPlan,
         search: searched,
     };