@agentshouse/mdruntime 0.1.0-cc294e382554d → 0.1.0-cc56db96f86ae

package/dist/read-side.js

@@ -1,8 +1,9 @@
-import { and, BasicIndex, configureIndexDevMode, count, createCollection, createLiveQueryCollection, deepEquals, eq, inArray, or, toArray, } from '@tanstack/db';
+import { and, coalesce, configureIndexDevMode, count, createCollection, createLiveQueryCollection, deepEquals, eq, inArray, not, or, } from '@tanstack/db';
 import { ARRAY_INDEX, parse, pointerSegments } from '@agentshouse/mdmodel';
+import { rowsOf, } from './observation.js';
 import { documentKey } from './rows.js';
-import { after, inUnits, ROW_WEIGHT } from './turn.js';
-import { beyond, cursorOf, MAXIMUM_WINDOW, windowed } from './windows.js';
+import { after, inUnits, ROW_WEIGHT, stepped } from './turn.js';
+import { beyond, cursorOf, FROM_START, MAXIMUM_WINDOW } from './windows.js';
 function every(expressions) {
     const [first, second, ...rest] = expressions;
     return second === undefined ? first : and(first, second, ...rest);
@@ -55,18 +56,21 @@ function conditioned(condition, row) {
 function narrowed(query, where, alias) {
     return where === null ? query : query.fn.where((row) => conditioned(where, row[alias]));
 }
-function reached(reach, room, document) {
-    return reach === null
-        ? []
-        : [or(inArray(room, [...reach.rooms]), inArray(document, [...reach.documents]))];
+function reached(reach, locked, room, document) {
+    return [
+        ...(reach === null
+            ? []
+            : [or(inArray(room, [...reach.rooms]), inArray(document, [...reach.documents]))]),
+        ...(locked.length === 0 ? [] : [not(inArray(room, [...locked]))]),
+    ];
 }
-function scoped(scope, row, rooms, reach, paths = null, folder = null) {
+function scoped(scope, row, rooms, reach, locked, paths = null, folder = null) {
     return every([
         eq(row.kind, 'document'),
         ...(rooms === null ? [] : [inArray(row.room, [...rooms])]),
         ...(paths === null ? [] : [inArray(row.path, [...paths])]),
         ...(folder === null ? [] : [eq(row.folder, folder)]),
-        ...reached(reach, row.room, row.key),
+        ...reached(reach, locked, row.room, row.key),
         ...(scope.types === null ? [] : [inArray(row.type, [...scope.types])]),
         ...(scope.pathPrefix === null ? [] : [inArray(scope.pathPrefix, row.directories)]),
     ]);
@@ -74,33 +78,42 @@ function scoped(scope, row, rooms, reach, paths = null, folder = null) {
 function keyed(key, row) {
     return key.of === 'field' ? row.values[key.field] : row[key.of];
 }
+function readied(live) {
+    return new Promise((resolve) => {
+        live.onFirstReady(resolve);
+    });
+}
+function streamed(value) {
+    return coalesce(value);
+}
 function live(query) {
     return createLiveQueryCollection({
         query: query,
         startSync: true,
+        gcTime: 0,
     });
 }
-function related(q, rows, scope, field, kept, reach) {
+function related(q, rows, scope, field, kept, reach, locked) {
     const matched = kept === 'source' ? 'target' : 'source';
     const from = q.from({ r: rows });
     const joined = from
         .where(({ r }) => every([eq(r.kind, 'reference'), ...(field === null ? [] : [inArray(field, r.forms)])]))
-        .innerJoin({ m: rows }, ({ r, m }) => eq(r[matched], m.key))
-        .where(({ m }) => scoped(scope, m, null, reach));
+        .innerJoin({ m: rows }, ({ r, m }) => eq(streamed(r[matched]), streamed(m.key)))
+        .where(({ m }) => scoped(scope, m, null, reach, locked));
     return narrowed(joined, scope.where, 'm')
         .groupBy(({ r }) => r[kept])
         .select(({ r }) => ({ key: r[kept] }));
 }
 function filtered(q, rows, plan) {
     const from = q.from({ d: rows });
-    let query = narrowed(from.where(({ d }) => scoped(plan.root, d, plan.rooms, plan.reach, plan.paths, plan.folder)), plan.root.where, 'd');
+    let query = narrowed(from.where(({ d }) => scoped(plan.root, d, plan.rooms, plan.reach, plan.locked, plan.paths, plan.folder)), plan.root.where, 'd');
     if (plan.hop !== null) {
-        const hop = related(q, rows, plan.hop, plan.hop.field, 'source', plan.reach);
-        query = query.innerJoin({ h: hop }, ({ d, h }) => eq(d.key, h.key));
+        const hop = related(q, rows, plan.hop, plan.hop.field, 'source', plan.reach, plan.locked);
+        query = query.innerJoin({ h: hop }, ({ d, h }) => eq(streamed(d.key), streamed(h.key)));
     }
     if (plan.backlinks !== null) {
-        const backlinks = related(q, rows, plan.backlinks, plan.backlinks.field, 'target', plan.reach);
-        query = query.innerJoin({ k: backlinks }, ({ d, k }) => eq(d.key, k.key));
+        const backlinks = related(q, rows, plan.backlinks, plan.backlinks.field, 'target', plan.reach, plan.locked);
+        query = query.innerJoin({ k: backlinks }, ({ d, k }) => eq(streamed(d.key), streamed(k.key)));
     }
     return query;
 }
@@ -139,10 +152,10 @@ function documents(q, rows, plan, shape) {
         query = query.fn.where(({ d }) => beyond([...shape.orderBy.map(({ key }) => storedValue(key, d)), d.key], cursor, orderings));
     }
     for (const { key, options } of shape.orderBy) {
-        query = query.orderBy(({ d }) => keyed(key, d), options);
+        query = query.orderBy(({ d }) => streamed(keyed(key, d)), options);
     }
     if (shape.orderBy.length > 0 || plan.limit !== null || plan.after !== null) {
-        query = query.orderBy(({ d }) => d.key, { stringSort: 'lexical' });
+        query = query.orderBy(({ d }) => streamed(d.key), { stringSort: 'lexical' });
     }
     if (plan.limit !== null)
         query = query.limit(plan.limit + 1);
@@ -154,18 +167,68 @@ function documents(q, rows, plan, shape) {
         ...Object.fromEntries(shape.orderBy.map(({ key }, index) => [`order${index}`, keyed(key, d)])),
         ...(shape.fields === null ? {} : { fields: d.fields }),
         ...(bodied(shape) ? { text: d.text } : {}),
-        ...(shape.backlinks ? { backlinks: toArray(backlinked(q, rows, d, plan.reach)) } : {}),
         ...(shape.change ? { change: d.change } : {}),
         ...(shape.fileId ? { fileId: d.fileId } : {}),
     }));
 }
-function backlinked(q, rows, document, reach) {
+function references(q, rows, reach, locked) {
     const from = q.from({ r: rows });
     return from
-        .where(({ r }) => every([eq(r.target, document.key), ...reached(reach, r.room, r.source)]))
-        .orderBy(({ r }) => r.path, { stringSort: 'lexical' })
-        .orderBy(({ r }) => r.field, { stringSort: 'lexical' })
-        .select(({ r }) => ({ path: r.path, type: r.type, field: r.field }));
+        .where(({ r }) => every([eq(r.kind, 'reference'), ...reached(reach, locked, r.room, r.source)]))
+        .select(({ r }) => ({
+        key: r.key,
+        target: r.target,
+        path: r.path,
+        type: r.type,
+        field: r.field,
+    }));
+}
+function lexical(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function linkOrder([leftKey, left], [rightKey, right]) {
+    return (lexical(left.path, right.path) || lexical(left.field, right.field) || lexical(leftKey, rightKey));
+}
+function linkAt(held, entry) {
+    let low = 0;
+    for (let high = held.length; low < high;) {
+        const middle = (low + high) >>> 1;
+        if (linkOrder(held[middle], entry) < 0)
+            low = middle + 1;
+        else
+            high = middle;
+    }
+    return low;
+}
+function targetOf(change) {
+    return change.value.target;
+}
+function relink(linked, { type, key, value }) {
+    const row = value;
+    const entry = [String(key), { path: row.path, type: row.type, field: row.field }];
+    const held = linked.get(row.target) ?? [];
+    const at = linkAt(held, entry);
+    const present = held[at]?.[0] === entry[0];
+    const shifted = (type === 'delete') === present ? held.length - at : 0;
+    if (type === 'delete') {
+        if (present)
+            held.splice(at, 1);
+    }
+    else if (present)
+        held[at] = entry;
+    else
+        held.splice(at, 0, entry);
+    if (held.length === 0)
+        linked.delete(row.target);
+    else
+        linked.set(row.target, held);
+    return shifted;
+}
+function relinked(linked, changes) {
+    return strides(changes, (change) => ROW_WEIGHT + relink(linked, change) / HELD_PER_WEIGHT);
+}
+function linking(links, received) {
+    return links.subscribeChanges(received, { includeInitialState: true });
 }
 function groups(q, stage, shape) {
     const keys = shape.groupBy.map((_, index) => `key${index}`);
@@ -186,13 +249,19 @@ function built(rows, plan) {
         const count = plan.total
             ? live((q) => filtered(q, rows, plan).select(({ d }) => ({ key: d.key })))
             : null;
+        const links = shape.backlinks
+            ? live((q) => references(q, rows, plan.reach, plan.locked))
+            : null;
         return {
             live: collection,
             count,
+            links,
             dispose: () => {
                 void collection.cleanup();
                 if (count !== null)
                     void count.cleanup();
+                if (links !== null)
+                    void links.cleanup();
             },
         };
     }
@@ -204,6 +273,7 @@ function built(rows, plan) {
     return {
         live: collection,
         count: null,
+        links: null,
         dispose: () => {
             void collection.cleanup();
             void stage.cleanup();
@@ -276,7 +346,7 @@ function cut(value, mask) {
 function selected(fields, pointers) {
     return (cut(fields, masked(pointers)) ?? {});
 }
-function documentOf(key, row, shape, bodies) {
+function documentOf(key, row, shape, bodies, linked) {
     if (row.type === null) {
         const binary = row.binary;
         return {
@@ -305,19 +375,52 @@ function documentOf(key, row, shape, bodies) {
         ...(shape.markdown && body !== null ? { markdown: body.content } : {}),
         ...(shape.backlinks
             ? {
-                backlinks: row.backlinks
+                backlinks: (linked.get(documentKey(row.room, row.path)) ?? [])
                     .slice(0, MAXIMUM_WINDOW)
-                    .map(({ path, type, field }) => ({ path, type, field })),
+                    .map(([, link]) => link),
             }
             : {}),
         ...(shape.change ? { change: row.change } : {}),
         ...(shape.fileId ? { fileId: row.fileId } : {}),
     };
 }
+function valueAt(row, path) {
+    let value = row;
+    for (const part of path)
+        value = value?.[part];
+    return value;
+}
 function weight(row) {
     return ROW_WEIGHT + (typeof row?.text === 'string' ? row.text.length : 0);
 }
 const HELD_PER_WEIGHT = 16;
+const STRIDE = ROW_WEIGHT * 16;
+function* strides(items, each) {
+    let spent = 0;
+    for (const item of items) {
+        spent += each(item);
+        if (spent < STRIDE)
+            continue;
+        yield spent;
+        spent = 0;
+    }
+    if (spent > 0)
+        yield spent;
+}
+function joined(plan) {
+    return (plan.hop !== null ||
+        plan.backlinks !== null ||
+        (plan.shape.kind === 'documents' && plan.shape.backlinks));
+}
+function admission(plan) {
+    if (joined(plan))
+        return () => true;
+    const rooms = plan.rooms === null ? null : new Set(plan.rooms);
+    const paths = plan.paths === null ? null : new Set(plan.paths);
+    return (row) => row.kind === 'document' &&
+        (rooms === null || rooms.has(row.room)) &&
+        (paths === null || paths.has(row.path));
+}
 function groupValues(shape, [key, row]) {
     return [...shape.orderBy.map(({ by }) => (by === 'count' ? row.count : row[`key${by}`])), key];
 }
@@ -330,23 +433,33 @@ function groupOrder(shape) {
         localeOptions: undefined,
     }));
 }
-function windowOf(plan, entries, total) {
+function* windowOf(plan, keys, valueOf, total, each) {
     const shape = plan.shape;
-    if (shape.kind === 'documents') {
-        return {
-            ...windowed(entries, plan.limit, plan.after?.text, ([, row]) => documentCursor(shape, row)),
-            total,
-        };
-    }
-    const cursor = plan.after?.values ?? null;
-    const orderings = groupOrder(shape);
-    const following = cursor === null
-        ? entries
-        : entries.filter((entry) => beyond(groupValues(shape, entry), cursor, orderings));
-    return {
-        ...windowed(following, plan.limit, plan.after?.text, (entry) => cursorOf(groupValues(shape, entry))),
-        total: plan.total ? entries.length : undefined,
-    };
+    const cursor = shape.kind === 'groups' ? (plan.after?.values ?? null) : null;
+    const orderings = shape.kind === 'groups' ? groupOrder(shape) : [];
+    let all = 0;
+    let following = 0;
+    let last;
+    yield* strides(keys, (key) => {
+        all += 1;
+        const admitted = shape.kind === 'documents' ||
+            cursor === null ||
+            beyond(groupValues(shape, [key, valueOf(key)]), cursor, orderings);
+        if (admitted)
+            following += 1;
+        if (!admitted || (plan.limit !== null && following > plan.limit))
+            return ROW_WEIGHT;
+        last = key;
+        return each(key);
+    });
+    const next = plan.limit === null || following <= plan.limit
+        ? undefined
+        : last === undefined
+            ? (plan.after?.text ?? FROM_START)
+            : shape.kind === 'documents'
+                ? documentCursor(shape, valueOf(last))
+                : cursorOf(groupValues(shape, [last, valueOf(last)]));
+    return { next, total: shape.kind === 'documents' ? total : plan.total ? all : undefined };
 }
 function framed(rows, { next, total }) {
     return {
@@ -355,27 +468,36 @@ function framed(rows, { next, total }) {
         ...(total === undefined ? {} : { total }),
     };
 }
-function rowsOfPlan(plan, entries, total, bodies, each = (row) => row) {
-    const shape = plan.shape;
-    const { kept, ...window } = windowOf(plan, entries, total);
-    const keys = kept.map(([key]) => key);
-    if (shape.kind === 'documents') {
-        const documents = [];
-        return after(inUnits(kept, ([key, row]) => {
-            documents.push(each(documentOf(key, row, shape, bodies)));
-            return weight(row);
-        }), () => ({ rows: framed({ documents }, window), keys, ...window }));
-    }
-    const groups = [];
-    return after(inUnits(kept, ([, row]) => {
-        groups.push(each({
-            key: shape.groupBy.map((_, index) => row[`key${index}`]),
-            count: row.count,
-        }));
-        return ROW_WEIGHT;
-    }), () => ({ rows: framed({ groups }, window), keys, ...window }));
+function rowOf(shape, [key, row], bodies, linked) {
+    if (shape.kind === 'documents')
+        return documentOf(key, row, shape, bodies, linked);
+    return { key: shape.groupBy.map((_, index) => row[`key${index}`]), count: row.count };
 }
-export function reconciled(previous, next) {
+function resultOf(grouped, rows, coverage, window) {
+    return grouped
+        ? framed({ groups: rows, coverage }, window)
+        : framed({ documents: rows, coverage }, window);
+}
+function* rowsOfPlan(plan, keys, valueOf, total, linked) {
+    const rows = [];
+    const window = yield* windowOf(plan, keys, valueOf, total, (key) => {
+        const value = valueOf(key);
+        rows.push(structuredClone(rowOf(plan.shape, [key, value], null, linked)));
+        return weight(value);
+    });
+    return plan.shape.kind === 'groups'
+        ? framed({ groups: rows }, window)
+        : framed({ documents: rows }, window);
+}
+function textOf(row) {
+    if (!('path' in row))
+        return 0;
+    let length = (row.markdown?.length ?? 0) + (row.preamble?.length ?? 0);
+    for (const section of Object.values(row.sections ?? {}))
+        length += section.length;
+    return length;
+}
+export function* reconciled(previous, next) {
     const before = previous.result;
     const after = next.result;
     if ('pending' in before || 'pending' in after) {
@@ -383,40 +505,67 @@ export function reconciled(previous, next) {
             return null;
         return { keyed: next, message: next };
     }
-    const earlier = 'groups' in before ? before.groups : before.documents;
-    const held = new Map(previous.keys.map((key, index) => [key, earlier[index]]));
+    const earlier = rowsOf(before);
+    const held = new Map();
+    yield* strides(previous.keys.entries(), ([index, key]) => {
+        held.set(key, earlier[index]);
+        return ROW_WEIGHT;
+    });
+    const later = rowsOf(after);
+    const present = new Set();
     const changed = [];
-    const rows = ('groups' in after ? after.groups : after.documents).map((row, index) => {
-        const key = next.keys[index];
+    const rows = [];
+    yield* strides(next.keys.entries(), ([index, key]) => {
+        const row = later[index];
         const kept = held.get(key);
+        present.add(key);
         if (kept !== undefined && deepEquals(kept, row))
-            return kept;
-        changed.push([key, row]);
-        return row;
+            rows.push(kept);
+        else {
+            changed.push([key, row]);
+            rows.push(row);
+        }
+        return ROW_WEIGHT + textOf(row);
+    });
+    const keyed = {
+        result: resultOf('groups' in after, rows, after.coverage, {
+            next: after.next,
+            total: after.total,
+        }),
+        keys: next.keys,
+    };
+    const message = yield* changeOf(previous, keyed, changed, (key) => present.has(key));
+    return message === null ? null : { keyed, message };
+}
+function* changeOf(previous, next, changed, present, dropped = () => { }) {
+    const before = previous.result;
+    const after = next.result;
+    const removed = [];
+    let at = 0;
+    let moved = false;
+    yield* strides(previous.keys.entries(), ([index, key]) => {
+        if (present(key, index)) {
+            moved ||= next.keys[at] !== key;
+            at += 1;
+        }
+        else {
+            removed.push(key);
+            dropped(key);
+        }
+        return ROW_WEIGHT;
     });
-    const present = new Set(next.keys);
-    const removed = previous.keys.filter((key) => !present.has(key));
-    const moved = reordered(previous.keys, next.keys, present);
+    moved ||= at !== next.keys.length;
     const covered = !deepEquals(before.coverage, after.coverage);
     const framing = reframed(before, after);
     if (changed.length === 0 && removed.length === 0 && !moved && !covered && framing === null) {
         return null;
     }
-    const window = { next: after.next, total: after.total };
     return {
-        keyed: {
-            result: 'groups' in after
-                ? framed({ groups: rows, coverage: after.coverage }, window)
-                : framed({ documents: rows, coverage: after.coverage }, window),
-            keys: next.keys,
-        },
-        message: {
-            changed,
-            removed,
-            order: moved ? next.keys : null,
-            ...(covered ? { coverage: after.coverage } : {}),
-            ...framing,
-        },
+        changed,
+        removed,
+        order: moved ? next.keys : null,
+        ...(covered ? { coverage: after.coverage } : {}),
+        ...framing,
     };
 }
 function reframed(before, after) {
@@ -426,38 +575,121 @@ function reframed(before, after) {
         ? null
         : { ...next, ...total };
 }
-function reordered(previous, next, present) {
-    let at = 0;
-    for (const key of previous) {
-        if (!present.has(key))
-            continue;
-        if (next[at] !== key)
-            return true;
-        at += 1;
-    }
-    return at !== next.length;
-}
 export function createReadSide() {
     configureIndexDevMode({ enabled: false });
-    let sync;
-    const rows = createCollection({
-        getKey: (row) => row.key,
-        gcTime: 0,
-        startSync: true,
-        autoIndex: 'eager',
-        defaultIndexType: BasicIndex,
-        sync: {
-            rowUpdateMode: 'full',
-            sync: (params) => {
-                sync = params;
-                params.markReady();
-            },
-        },
-    });
+    const rows = new Map();
+    const roomPaths = new Map();
+    const views = new Set();
     const shared = new Map();
     let writing = false;
     let publishing = null;
     let closed = false;
+    function viewOf(plan) {
+        let sync;
+        const view = {
+            plan,
+            rows: createCollection({
+                getKey: (row) => row.key,
+                gcTime: 0,
+                startSync: true,
+                autoIndex: 'off',
+                sync: {
+                    rowUpdateMode: 'full',
+                    sync: (params) => {
+                        sync = params;
+                    },
+                },
+            }),
+            sync,
+            admits: admission(plan),
+            stopped: false,
+            waiting: null,
+        };
+        views.add(view);
+        return view;
+    }
+    function freeze(view) {
+        view.waiting ??= new Map();
+    }
+    function thaw(view) {
+        const waiting = view.waiting;
+        if (waiting === null)
+            return;
+        let unit = new Map();
+        return after(inUnits(waiting, ([key, row]) => {
+            waiting.delete(key);
+            unit.set(key, row);
+            const was = view.rows.get(key);
+            return weight(row ?? undefined) + charged(view, was, row ?? undefined);
+        }, () => {
+            synced(view, unit);
+            unit = new Map();
+        }), () => {
+            if (waiting.size > 0)
+                return thaw(view);
+            view.waiting = null;
+        });
+    }
+    function dropped(view) {
+        view.stopped = true;
+        views.delete(view);
+        void view.rows.cleanup();
+    }
+    function* offered(view) {
+        const { plan } = view;
+        const { rooms, paths } = plan;
+        if (joined(plan) || rooms === null || paths === null) {
+            for (const key of rows.keys()) {
+                if (view.stopped)
+                    return;
+                yield key;
+            }
+            return;
+        }
+        for (const room of rooms) {
+            for (const path of paths) {
+                if (view.stopped)
+                    return;
+                yield documentKey(room, path);
+            }
+        }
+    }
+    function written(view, writes) {
+        if (view.waiting === null)
+            synced(view, writes);
+        else
+            for (const [key, row] of writes)
+                view.waiting.set(key, row);
+    }
+    function synced(view, writes) {
+        if (view.stopped || writes.size === 0)
+            return;
+        view.sync.begin();
+        for (const [key, row] of writes) {
+            if (row !== null) {
+                view.sync.write({ type: view.rows.has(key) ? 'update' : 'insert', value: row });
+            }
+            else if (view.rows.has(key))
+                view.sync.write({ type: 'delete', key });
+        }
+        view.sync.commit();
+    }
+    function filled(view) {
+        let unit = new Map();
+        return after(inUnits(offered(view), (key) => {
+            const row = rows.get(key);
+            if (row === undefined || !view.admits(row) || view.rows.has(key))
+                return ROW_WEIGHT;
+            unit.set(key, row);
+            return weight(row) + charged(view, undefined, row);
+        }, () => {
+            written(view, unit);
+            unit = new Map();
+        }), () => {
+            if (!view.stopped)
+                view.sync.markReady();
+        });
+    }
     function searched(row) {
         const read = row.text === null ? null : parse(row.text, row.path);
         if (read === null || !read.ok)
@@ -471,80 +703,116 @@ export function createReadSide() {
             sections: read.document.sections,
         };
     }
-    function kept(entry, key, row) {
-        const held = entry.rows.get(key);
-        if (held !== undefined && deepEquals(held, row))
-            return;
-        entry.rows.set(key, row);
-    }
     function counted(entry) {
         return entry.built.count === null ? undefined : entry.built.count.size;
     }
-    function rebuilt(entry) {
-        return after(rowsOfPlan(entry.plan, [...entry.built.live.entries()], counted(entry), entry.bodies), ({ rows: found, keys, next, total }) => {
-            const present = new Set(keys);
-            for (const key of [...entry.rows.keys()])
-                if (!present.has(key))
-                    entry.rows.delete(key);
-            for (const key of [...entry.bodies.keys()]) {
-                if (!present.has(key))
-                    entry.bodies.delete(key);
+    function* delivery(entry, whole, dirty, touched, relinking, coverage) {
+        const { plan } = entry;
+        const live = entry.built.live;
+        const valueOf = (key) => live.get(key);
+        const previous = entry.result;
+        const answered = previous === undefined || 'pending' in previous.result ? null : previous;
+        const answeredKeys = answered?.keys ?? [];
+        const answeredRows = answered === null ? [] : rowsOf(answered.result);
+        const matched = new Uint8Array(answeredKeys.length);
+        const generation = (entry.generation += 1);
+        const keys = [];
+        const changed = [];
+        const rows = [];
+        let at = 0;
+        const touches = (key) => {
+            const { room, path } = valueOf(key);
+            return touched.has(documentKey(room, path));
+        };
+        const behind = (key) => !live.has(key) || entry.rows.get(key)?.at === generation;
+        yield* relinked(entry.linked, relinking);
+        const window = yield* windowOf(plan, live.keys(), valueOf, counted(entry), (key) => {
+            let skipped = 0;
+            while (skipped < STRIDE &&
+                at < answeredKeys.length &&
+                answeredKeys[at] !== key &&
+                behind(answeredKeys[at])) {
+                at += 1;
+                skipped += ROW_WEIGHT;
+            }
+            let holder;
+            let prior;
+            if (at < answeredKeys.length && answeredKeys[at] === key) {
+                prior = answeredRows[at];
+                matched[at] = 1;
+                at += 1;
             }
-            keys.forEach((key, index) => kept(entry, key, rowAt(found, index)));
-            return { keys, next, total };
+            else {
+                holder = entry.rows.get(key);
+                prior = holder?.row;
+                if (holder !== undefined)
+                    holder.at = generation;
+            }
+            let row = prior;
+            let spent = ROW_WEIGHT;
+            if (row === undefined || whole || dirty.has(key) || (touched.size > 0 && touches(key))) {
+                const value = valueOf(key);
+                const renewed = rowOf(plan.shape, [key, value], entry.bodies, entry.linked);
+                if (row === undefined || !deepEquals(row, renewed))
+                    row = renewed;
+                spent = weight(value);
+            }
+            if (row !== prior) {
+                holder ??= entry.rows.get(key);
+                if (holder === undefined)
+                    entry.rows.set(key, { row, at: generation });
+                else
+                    holder.row = row;
+                changed.push([key, row]);
+            }
+            keys.push(key);
+            rows.push(row);
+            return skipped + spent;
         });
-    }
-    function refreshed(entry, dirty) {
-        const shape = entry.plan.shape;
-        const { kept: inWindow, next, total, } = windowOf(entry.plan, [...entry.built.live.entries()], counted(entry));
-        const keys = inWindow.map(([key]) => key);
-        const present = new Set(keys);
-        for (const key of [...entry.rows.keys()]) {
-            if (present.has(key))
-                continue;
+        const result = {
+            result: resultOf(plan.shape.kind === 'groups', rows, coverage, window),
+            keys,
+        };
+        if (previous === undefined)
+            return { result, message: null };
+        if (answered === null)
+            return { result, message: result };
+        const message = yield* changeOf(answered, result, changed, (key, index) => matched[index] === 1 || entry.rows.get(key)?.at === generation, (key) => {
             entry.rows.delete(key);
             entry.bodies.delete(key);
-        }
-        const values = inWindow.filter(([key]) => dirty.has(key) || !entry.rows.has(key));
-        const done = inUnits(values, ([key, value]) => {
-            kept(entry, key, documentOf(key, value, shape, entry.bodies));
-            return weight(value);
         });
-        return after(done, () => ({ keys, next, total }));
+        return { result, message };
     }
-    function current(entry, found, coverage) {
-        const rows = found.keys.map((key) => entry.rows.get(key));
-        return {
-            result: entry.plan.shape.kind === 'groups'
-                ? framed({ groups: rows, coverage }, found)
-                : framed({ documents: rows, coverage }, found),
-            keys: found.keys,
-        };
-    }
-    function delivered(entry, next) {
-        const previous = entry.result;
-        entry.result = next;
-        const message = previous === undefined ? null : changedFrom(previous, next);
+    function delivered(entry, { result, message }) {
+        entry.result = result;
         if (message !== null)
             for (const listener of entry.ready)
                 listener(message);
         for (const listener of entry.fresh) {
             entry.ready.add(listener);
-            listener(next);
+            listener(result);
         }
         entry.fresh.clear();
     }
     function run(entry) {
         while (entry.listeners.size > 0 &&
-            (entry.whole || entry.dirty.size > 0 || entry.stale || entry.fresh.size > 0)) {
+            (entry.whole ||
+                entry.dirty.size > 0 ||
+                entry.touched.size > 0 ||
+                entry.stale ||
+                entry.fresh.size > 0)) {
             entry.stale = false;
             const cover = entry.cover();
             if ('pending' in cover) {
                 entry.whole = true;
                 entry.dirty.clear();
+                entry.touched.clear();
                 entry.rows.clear();
                 entry.bodies.clear();
-                delivered(entry, { result: cover, keys: [] });
+                const result = { result: cover, keys: [] };
+                const previous = entry.result;
+                const same = previous === undefined || deepEquals(previous.result, cover);
+                delivered(entry, { result, message: same ? null : result });
                 break;
             }
             const whole = entry.whole ||
@@ -552,23 +820,38 @@ export function createReadSide() {
                 entry.result === undefined ||
                 'pending' in entry.result.result;
             const dirty = entry.dirty;
+            const touched = entry.touched;
+            const relinking = entry.relinking;
             entry.dirty = new Set();
+            entry.touched = new Set();
+            entry.relinking = [];
             entry.whole = false;
-            const found = whole ? rebuilt(entry) : refreshed(entry, dirty);
-            if (found instanceof Promise) {
-                void found
-                    .then((window) => {
+            freeze(entry.view);
+            let made;
+            try {
+                made = stepped(delivery(entry, whole, dirty, touched, relinking, cover.coverage));
+            }
+            catch (error) {
+                void thaw(entry.view);
+                throw error;
+            }
+            if (made instanceof Promise) {
+                void made
+                    .then((delivering) => {
                     if (entry.listeners.size > 0)
-                        delivered(entry, current(entry, window, cover.coverage));
+                        delivered(entry, delivering);
+                })
+                    .finally(() => thaw(entry.view))
+                    .then(() => {
                     entry.running = false;
                     wake(entry);
-                })
-                    .catch(() => {
+                }, () => {
                     entry.running = false;
                 });
                 return;
             }
-            delivered(entry, current(entry, found, cover.coverage));
+            void thaw(entry.view);
+            delivered(entry, made);
         }
         entry.running = false;
     }
@@ -599,34 +882,84 @@ export function createReadSide() {
             wake(entry);
         }
     }
+    function listed(key, row) {
+        const held = rows.get(key) ?? row;
+        if (held?.kind !== 'document')
+            return;
+        const paths = roomPaths.get(held.room) ?? new Set();
+        if (row?.kind === 'document' && row.type !== null)
+            paths.add(key);
+        else
+            paths.delete(key);
+        if (paths.size === 0)
+            roomPaths.delete(held.room);
+        else
+            roomPaths.set(held.room, paths);
+    }
     function committed(next) {
         if (closed || next.size === 0)
             return;
-        sync.begin();
         for (const [key, row] of next) {
-            if (row === null) {
-                if (rows.has(key))
-                    sync.write({ type: 'delete', key });
-                continue;
+            listed(key, row);
+            if (row === null)
+                rows.delete(key);
+            else
+                rows.set(key, row);
+        }
+        for (const view of views) {
+            const writes = new Map();
+            for (const [key, row] of next) {
+                if (row !== null && view.admits(row))
+                    writes.set(key, row);
+                else if (view.rows.has(key) || view.waiting?.has(key))
+                    writes.set(key, null);
             }
-            sync.write({ type: rows.has(key) ? 'update' : 'insert', value: row });
+            written(view, writes);
         }
-        sync.commit();
+    }
+    function moved({ plan }, before, after) {
+        const shape = plan.shape;
+        if (shape.kind === 'documents' && plan.limit === null && shape.orderBy.length > 0) {
+            return true;
+        }
+        const keys = shape.kind === 'documents' ? shape.orderBy.map(({ key }) => key) : shape.groupBy;
+        for (const key of keys) {
+            const path = key.of === 'field' ? ['values', key.field] : [key.of];
+            if (!deepEquals(valueAt(before, path), valueAt(after, path)))
+                return true;
+        }
+        return false;
+    }
+    function charged(view, was, row) {
+        const kept = was !== undefined && view.admits(was);
+        const admitted = row !== undefined && view.admits(row);
+        if (!kept && !admitted)
+            return 0;
+        const shifted = view.rows.size / HELD_PER_WEIGHT;
+        return shifted + (kept && admitted && !moved(view, was, row) ? 0 : shifted);
+    }
+    function charges(was, row) {
+        let spent = 0;
+        for (const view of views)
+            spent += charged(view, was, row);
+        return spent;
     }
     function published(set) {
         let unit = new Map();
         return inUnits(set.documents, (change) => {
             if (closed)
                 return 0;
-            const shifted = rows.size / HELD_PER_WEIGHT;
+            const { before, after } = change;
+            const kept = new Set(after.map((row) => row.key));
+            const held = new Map(before.map((row) => [row.key, row]));
             let spent = 0;
-            for (const row of change.before) {
+            for (const row of before) {
                 unit.set(row.key, null);
-                spent += ROW_WEIGHT + shifted;
+                spent += ROW_WEIGHT + (kept.has(row.key) ? 0 : charges(row, undefined));
             }
-            for (const row of change.after) {
+            for (const row of after) {
                 unit.set(row.key, row);
-                spent += weight(row) + shifted;
+                spent += weight(row) + charges(held.get(row.key), row);
             }
             return spent;
         }, () => {
@@ -657,8 +990,8 @@ export function createReadSide() {
     return {
         apply(set, next = () => { }) {
             const pending = publishing;
-            if (pending !== null) {
-                return publishedLater(pending.then(() => after(published(set), next)));
+            if (pending !== null || set instanceof Promise) {
+                return publishedLater(Promise.all([set, pending]).then(([held]) => after(published(held), () => next(held))));
             }
             writing = true;
             let applied;
@@ -669,48 +1002,83 @@ export function createReadSide() {
                 writing = false;
             }
             if (applied instanceof Promise)
-                return publishedLater(applied.then(next));
+                return publishedLater(applied.then(() => next(set)));
             released();
-            next();
+            next(set);
         },
         recover,
         settled,
         async query(plan, at) {
-            const { live: collection, count, dispose } = built(rows, plan);
+            const view = viewOf(plan);
+            const linked = new Map();
+            let made = null;
+            let following = null;
             try {
-                await collection.toArrayWhenReady();
-                if (count !== null)
-                    await count.toArrayWhenReady();
+                const filling = filled(view);
+                made = built(view.rows, plan);
+                const { live: collection, count, links } = made;
+                if (links !== null) {
+                    following = linking(links, (changes) => {
+                        for (const change of changes)
+                            relink(linked, change);
+                    });
+                }
+                await filling;
+                for (const one of [collection, count, links])
+                    if (one !== null)
+                        await readied(one);
                 await settled();
-                const entries = [...collection.entries()];
+                freeze(view);
+                following?.unsubscribe();
+                following = null;
                 const held = at();
-                const found = await rowsOfPlan(plan, entries, count?.size, null, structuredClone);
-                return { rows: found.rows, at: held };
+                const rows = await stepped(rowsOfPlan(plan, collection.keys(), (key) => collection.get(key), count?.size, linked));
+                return { rows, at: held };
             }
             finally {
-                dispose();
+                following?.unsubscribe();
+                made?.dispose();
+                dropped(view);
             }
         },
         watch(plan, identity, cover, deliverTo) {
             let entry = shared.get(identity);
             if (entry === undefined) {
+                const view = viewOf(plan);
+                void filled(view);
                 const created = {
                     plan,
-                    built: built(rows, plan),
+                    view,
+                    built: built(view.rows, plan),
                     cover,
                     result: undefined,
                     rows: new Map(),
+                    generation: 0,
                     bodies: new Map(),
                     listeners: new Set(),
                     fresh: new Set(),
                     ready: new Set(),
                     subscription: null,
                     counting: null,
+                    linking: null,
+                    linked: new Map(),
+                    relinking: [],
+                    touched: new Set(),
                     whole: true,
                     dirty: new Set(),
                     stale: false,
                     running: false,
                 };
+                if (created.built.links !== null) {
+                    created.linking = linking(created.built.links, (changes) => {
+                        for (const change of changes) {
+                            created.relinking.push(change);
+                            created.touched.add(targetOf(change));
+                        }
+                        if (created.subscription !== null)
+                            queueMicrotask(() => wake(created));
+                    });
+                }
                 created.built.live.on('status:change', ({ status }) => {
                     if (status !== 'cleaned-up' || created.listeners.size > 0)
                         return;
@@ -722,7 +1090,12 @@ export function createReadSide() {
             }
             const observed = entry;
             observed.listeners.add(deliverTo);
-            const ready = (then) => observed.built.live.onFirstReady(() => observed.built.count === null ? then() : observed.built.count.onFirstReady(then));
+            const ready = (then) => observed.built.live.onFirstReady(() => {
+                const linked = () => observed.built.links === null ? then() : observed.built.links.onFirstReady(then);
+                return observed.built.count === null
+                    ? linked()
+                    : observed.built.count.onFirstReady(linked);
+            });
             const settled = () => {
                 if (observed.built.count === null)
                     wake(observed);
@@ -760,22 +1133,30 @@ export function createReadSide() {
                 observed.subscription = null;
                 observed.counting?.unsubscribe();
                 observed.counting = null;
+                observed.linking?.unsubscribe();
+                observed.linking = null;
                 observed.result = undefined;
                 observed.rows.clear();
                 observed.bodies.clear();
                 observed.dirty.clear();
+                observed.touched.clear();
                 observed.whole = true;
                 if (shared.get(identity) === observed)
                     shared.delete(identity);
                 observed.built.dispose();
+                dropped(observed.view);
             };
         },
-        room(room) {
-            return rows.toArray.flatMap((row) => row.kind !== 'document' || row.type === null || row.room !== room ? [] : [row.path]);
+        *room(room) {
+            for (const key of roomPaths.get(room) ?? []) {
+                const row = rows.get(key);
+                if (row?.kind === 'document')
+                    yield row.path;
+            }
         },
         document(room, path) {
             const row = rows.get(documentKey(room, path));
-            return row === undefined || row.type === null ? null : searched(row);
+            return row?.kind !== 'document' || row.type === null ? null : searched(row);
         },
         close() {
             closed = true;
@@ -785,42 +1166,14 @@ export function createReadSide() {
                 entry.ready.clear();
                 entry.subscription?.unsubscribe();
                 entry.counting?.unsubscribe();
+                entry.linking?.unsubscribe();
                 entry.built.dispose();
             }
             shared.clear();
-            void rows.cleanup();
+            for (const view of [...views])
+                dropped(view);
+            rows.clear();
+            roomPaths.clear();
         },
     };
 }
-function changedFrom(previous, next) {
-    const before = previous.result;
-    const after = next.result;
-    if ('pending' in before || 'pending' in after)
-        return deepEquals(before, after) ? null : next;
-    const earlier = 'groups' in before ? before.groups : before.documents;
-    const held = new Map(previous.keys.map((key, index) => [key, earlier[index]]));
-    const later = 'groups' in after ? after.groups : after.documents;
-    const changed = [];
-    next.keys.forEach((key, index) => {
-        if (held.get(key) !== later[index])
-            changed.push([key, later[index]]);
-    });
-    const present = new Set(next.keys);
-    const removed = previous.keys.filter((key) => !present.has(key));
-    const moved = reordered(previous.keys, next.keys, present);
-    const covered = !deepEquals(before.coverage, after.coverage);
-    const framing = reframed(before, after);
-    if (changed.length === 0 && removed.length === 0 && !moved && !covered && framing === null) {
-        return null;
-    }
-    return {
-        changed,
-        removed,
-        order: moved ? next.keys : null,
-        ...(covered ? { coverage: after.coverage } : {}),
-        ...framing,
-    };
-}
-function rowAt(rows, index) {
-    return 'groups' in rows ? rows.groups[index] : rows.documents[index];
-}