@agentshouse/mdruntime 0.1.0-cefbc8bb8a1c0 → 0.1.0-cf04ede9fb796

package/dist/stored.js

@@ -1,22 +1,22 @@
+import { instantKey, sql } from '@agentshouse/mdmodel';
 import { composedOf } from './collection.js';
-import { indexDocument, unindexAll, unindexDocuments } from './fulltext.js';
-import { described, derived } from './rows.js';
+import { described, derived, heldOf } from './rows.js';
 export const REPLICA_TABLES = [
-    'CREATE TABLE replica_keys (key INTEGER PRIMARY KEY AUTOINCREMENT, locator TEXT NOT NULL, transfer TEXT NOT NULL, fingerprint TEXT NOT NULL, rederiving INTEGER, searching INTEGER)',
-    'CREATE TABLE replica_rooms (locator TEXT PRIMARY KEY, key INTEGER NOT NULL, record TEXT NOT NULL) WITHOUT ROWID',
-    'CREATE TABLE replica_entries (id INTEGER PRIMARY KEY, key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, visible INTEGER NOT NULL, present INTEGER NOT NULL, queried INTEGER NOT NULL, binary TEXT, revision TEXT, file_id TEXT, change TEXT, written TEXT, type TEXT, declared TEXT, folder TEXT NOT NULL, fields TEXT, content TEXT)',
+    'CREATE TABLE replica_keys (key INTEGER PRIMARY KEY AUTOINCREMENT, locator TEXT NOT NULL, transfer TEXT NOT NULL, fingerprint TEXT NOT NULL, rederiving INTEGER)',
+    'CREATE TABLE replica_rooms (locator TEXT PRIMARY KEY, key INTEGER NOT NULL, record TEXT NOT NULL, documents INTEGER NOT NULL DEFAULT 0) WITHOUT ROWID',
+    "CREATE INDEX replica_rooms_model ON replica_rooms (json_extract(record, '$.model.snapshot.identity'), json_extract(record, '$.model.snapshot.version'))",
+    'CREATE TABLE replica_models (identity TEXT NOT NULL, version TEXT NOT NULL, schema TEXT NOT NULL, PRIMARY KEY (identity, version)) WITHOUT ROWID',
+    "CREATE TABLE replica_entries (id INTEGER PRIMARY KEY, key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, visible INTEGER NOT NULL, present INTEGER NOT NULL, queried INTEGER NOT NULL, binary TEXT, revision TEXT, file_id TEXT, change TEXT, written TEXT, type TEXT, declared TEXT, folder TEXT NOT NULL, fields TEXT, links TEXT, part TEXT, created TEXT, created_instant TEXT, changed_instant TEXT, bytes INTEGER GENERATED ALWAYS AS (CASE WHEN binary IS NULL THEN length(CAST(content AS BLOB)) ELSE binary ->> '$.byteLength' END) STORED, content TEXT)",
     'CREATE UNIQUE INDEX replica_entries_path ON replica_entries (key, layer, path)',
     'CREATE INDEX replica_entries_visible ON replica_entries (key, visible, present, path, binary)',
     'CREATE INDEX replica_entries_folder ON replica_entries (key, folder, path)',
     'CREATE INDEX replica_entries_type ON replica_entries (key, type, visible, present, queried, folder)',
     'CREATE INDEX replica_entries_key ON replica_entries (key)',
-    'CREATE TABLE replica_values (doc INTEGER NOT NULL, keys TEXT NOT NULL, places TEXT NOT NULL, item INTEGER NOT NULL, kind TEXT NOT NULL, value, size INTEGER, PRIMARY KEY (doc, keys, places, item)) WITHOUT ROWID',
+    sql.VALUE_TABLE,
     'CREATE TABLE replica_links (doc INTEGER NOT NULL, key INTEGER NOT NULL, kind TEXT NOT NULL, keys TEXT NOT NULL, places TEXT NOT NULL, target TEXT NOT NULL, field TEXT NOT NULL, PRIMARY KEY (doc, kind, keys, places, target)) WITHOUT ROWID',
     'CREATE INDEX replica_links_target ON replica_links (key, target, kind, keys)',
-    'CREATE TABLE replica_folders (key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, present INTEGER NOT NULL, PRIMARY KEY (key, layer, path)) WITHOUT ROWID',
+    'CREATE TABLE replica_folders (key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, present INTEGER NOT NULL, part TEXT, PRIMARY KEY (key, layer, path)) WITHOUT ROWID',
     'CREATE TABLE replica_erasing (key INTEGER PRIMARY KEY, locator TEXT NOT NULL)',
-    'CREATE TABLE replica_received (key TEXT PRIMARY KEY, head TEXT NOT NULL) WITHOUT ROWID',
-    'CREATE TABLE replica_received_parts (key TEXT NOT NULL, part INTEGER NOT NULL, value TEXT NOT NULL, PRIMARY KEY (key, part)) WITHOUT ROWID',
     'CREATE TABLE replica_facts (name TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID',
     'CREATE TABLE replica_presentation (app TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID',
 ];
@@ -31,8 +31,7 @@ const ABSENT = {
     queried: false,
     folder: '',
     fields: null,
-    values: [],
-    links: [],
+    links: null,
 };
 function text(value) {
     return typeof value === 'string' ? value : null;
@@ -53,17 +52,11 @@ function dropDerived(connection, doc) {
     connection.run('DELETE FROM replica_values WHERE doc = ?', [doc]);
     connection.run('DELETE FROM replica_links WHERE doc = ?', [doc]);
 }
-function scalarText(value) {
-    if (value === Infinity)
-        return 'Infinity';
-    if (value === -Infinity)
-        return '-Infinity';
-    return JSON.stringify(value);
-}
-function writeDerived(connection, doc, key, made) {
+function writeDerived(connection, doc, key, path, held, composition) {
+    const made = derived(path, held, composition);
     if (made.values.length > 0) {
-        const rows = made.values.map((value) => `[${JSON.stringify(value.keys)},${JSON.stringify(value.places)},${value.item},${JSON.stringify(value.kind)},${scalarText(value.value)},${JSON.stringify(value.size)}]`);
-        connection.run('INSERT INTO replica_values (doc, keys, places, item, kind, value, size) SELECT ?, value ->> 0, value ->> 1, value ->> 2, value ->> 3, value ->> 4, value ->> 5 FROM json_each(?)', [doc, `[${rows.join(',')}]`]);
+        const inserted = sql.insertedValues(doc, made.values);
+        connection.run(inserted.sql, inserted.bind);
     }
     if (made.links.length > 0) {
         connection.run('INSERT OR IGNORE INTO replica_links (doc, key, kind, keys, places, target, field) SELECT ?, ?, value ->> 0, value ->> 1, value ->> 2, value ->> 3, value ->> 4 FROM json_each(?)', [
@@ -77,27 +70,15 @@ function entryId(connection, key, layer, path) {
     const [row] = connection.rows('SELECT id FROM replica_entries WHERE key = ? AND layer = ? AND path = ?', [key, layer, path]);
     return row === undefined ? null : number(row.id);
 }
-function words() {
-    return { dropped: [], made: [] };
-}
-function wordsWritten(connection, owed) {
-    unindexDocuments(connection, owed.dropped);
-    for (const { id, path, content } of owed.made)
-        indexDocument(connection, id, path, content);
-}
-function removeEntry(connection, key, layer, path, owed) {
+function removeEntry(connection, key, layer, path) {
     const id = entryId(connection, key, layer, path);
     if (id === null)
         return;
     dropDerived(connection, id);
-    owed.dropped.push(id);
     connection.run('DELETE FROM replica_entries WHERE id = ?', [id]);
 }
-function heldIds(connection, sql, bind) {
-    return connection.rows(sql, bind).map((row) => number(row.id));
-}
-function insertEntry(connection, key, layer, entry, present, made) {
-    connection.run('INSERT INTO replica_entries (key, layer, path, visible, present, queried, content, binary, revision, file_id, change, written, type, declared, folder, fields) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [
+function insertEntry(connection, key, layer, entry, present, made, part = null) {
+    connection.run('INSERT INTO replica_entries (key, layer, path, visible, present, queried, content, binary, revision, file_id, created, created_instant, change, changed_instant, written, type, declared, folder, fields, links, part) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [
         key,
         layer,
         entry.path,
@@ -108,32 +89,34 @@ function insertEntry(connection, key, layer, entry, present, made) {
         entry.binary === null ? null : JSON.stringify(entry.binary),
         entry.revision,
         entry.fileId,
+        entry.created,
+        instantKey(entry.created),
         entry.change === null ? null : JSON.stringify(entry.change),
+        instantKey(entry.change?.time),
         entry.written,
         made.type,
         made.declared,
         made.folder,
         made.fields,
+        made.links,
+        part,
     ]);
 }
 function textOf(entry) {
     return entry.binary === null ? entry.content : null;
 }
-function putEntry(connection, key, layer, entry, present, composition, owed, searched = true) {
-    removeEntry(connection, key, layer, entry.path, owed);
+function putEntry(connection, key, layer, entry, present, composition) {
+    removeEntry(connection, key, layer, entry.path);
     const content = textOf(entry);
-    const made = present ? derived(entry.path, content, composition) : ABSENT;
-    insertEntry(connection, key, layer, entry, present, made);
+    insertEntry(connection, key, layer, entry, present, present ? described(entry.path, content, entry.facts) : ABSENT);
     if (!present)
         return;
     const id = entryId(connection, key, layer, entry.path);
-    writeDerived(connection, id, key, made);
-    if (searched && content !== null && made.queried)
-        owed.made.push({ id, path: entry.path, content });
+    writeDerived(connection, id, key, entry.path, heldOf(entry.path, content, entry.facts), composition);
 }
-function placeEntry(connection, key, entry, owed) {
-    removeEntry(connection, key, CONFIRMED, entry.path, owed);
-    insertEntry(connection, key, CONFIRMED, entry, true, described(entry.path, textOf(entry)));
+function placeEntry(connection, key, entry, part) {
+    removeEntry(connection, key, CONFIRMED, entry.path);
+    insertEntry(connection, key, CONFIRMED, entry, true, described(entry.path, textOf(entry), entry.facts), part);
 }
 function revealed(connection, key, touched) {
     if (touched.length === 0)
@@ -142,7 +125,7 @@ function revealed(connection, key, touched) {
 }
 function newKey(connection, locator, transfer, fingerprint, built) {
     const owed = built ? null : 0;
-    connection.run('INSERT INTO replica_keys (locator, transfer, fingerprint, rederiving, searching) VALUES (?, ?, ?, ?, ?)', [locator, transfer, fingerprint, owed, owed]);
+    connection.run('INSERT INTO replica_keys (locator, transfer, fingerprint, rederiving) VALUES (?, ?, ?, ?)', [locator, transfer, fingerprint, owed]);
     const [row] = connection.rows('SELECT last_insert_rowid() AS key');
     return number(row.key);
 }
@@ -155,36 +138,96 @@ function recomposed(connection, key, fingerprint) {
 }
 export function entered(connection, locator, composition, rows) {
     const key = newKey(connection, locator, '', composedOf(composition).fingerprint, true);
-    const owed = words();
-    for (const entry of rows) {
-        putEntry(connection, key, CONFIRMED, entry, true, composition, owed, false);
-    }
-    wordsWritten(connection, owed);
+    for (const entry of rows)
+        putEntry(connection, key, CONFIRMED, entry, true, composition);
     return key;
 }
 export function stage(connection, request) {
+    if (request.key === null) {
+        for (const row of connection.rows(STAGED_KEYS, [request.locator])) {
+            erasing(connection, number(row.key), request.locator);
+        }
+    }
     const key = request.key ??
         newKey(connection, request.locator, request.transfer, request.fingerprint, false);
     recomposed(connection, key, request.fingerprint);
-    connection.run('UPDATE replica_keys SET rederiving = coalesce(rederiving, last), searching = coalesce(searching, last) FROM (SELECT coalesce(max(id), 0) AS last FROM replica_entries) WHERE key = ?', [key]);
-    const owed = words();
+    owing(connection, key);
     for (const entry of request.entries)
-        placeEntry(connection, key, entry, owed);
+        placeEntry(connection, key, entry, request.part);
     for (const path of request.folders) {
-        connection.run('INSERT OR REPLACE INTO replica_folders (key, layer, path, present) VALUES (?, ?, ?, 1)', [key, CONFIRMED, path]);
+        connection.run('INSERT OR REPLACE INTO replica_folders (key, layer, path, present, part) VALUES (?, ?, ?, 1, ?)', [key, CONFIRMED, path, request.part]);
     }
-    wordsWritten(connection, owed);
     return key;
 }
+function owing(connection, key) {
+    connection.run('UPDATE replica_keys SET rederiving = coalesce(rederiving, last) FROM (SELECT coalesce(max(id), 0) AS last FROM replica_entries) WHERE key = ?', [key]);
+}
+const STAGED_KEYS = `SELECT k.key, k.transfer FROM replica_keys k WHERE k.locator = ?
+  AND NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
+  AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)
+  ORDER BY k.key DESC`;
+const UNPARTED = `EXISTS (SELECT 1 FROM replica_entries e WHERE e.key = k.key AND e.part IS NULL)
+  OR EXISTS (SELECT 1 FROM replica_folders f WHERE f.key = k.key AND f.part IS NULL)`;
+function partsOf(connection, key, layer) {
+    const scoped = layer === null ? '' : ` AND layer = ${layer}`;
+    return new Set(connection
+        .rows(`SELECT part FROM replica_entries WHERE key = ?${scoped} AND part IS NOT NULL UNION SELECT part FROM replica_folders WHERE key = ?${scoped} AND part IS NOT NULL`, [key, key])
+        .map((row) => text(row.part)));
+}
+function resumable(connection, locator) {
+    let kept = null;
+    for (const row of connection.rows(STAGED_KEYS, [locator])) {
+        const key = number(row.key);
+        const parted = kept === null &&
+            connection.rows(`SELECT 1 FROM replica_keys k WHERE k.key = ? AND NOT (${UNPARTED})`, [key])
+                .length > 0;
+        if (parted)
+            kept = { key, transfer: text(row.transfer) };
+        else
+            erasing(connection, key, locator);
+    }
+    return kept;
+}
+export function resume(connection, request) {
+    const wanted = new Set(request.parts);
+    const held = resumable(connection, request.locator);
+    const staged = held === null ? new Set() : partsOf(connection, held.key, null);
+    const dropped = [...staged].filter((part) => !wanted.has(part));
+    if (held !== null && dropped.length > 0) {
+        const named = JSON.stringify(dropped);
+        const rows = `SELECT id FROM replica_entries WHERE key = ? AND part IN (SELECT value FROM json_each(?))`;
+        for (const table of ['replica_values', 'replica_links']) {
+            connection.run(`DELETE FROM ${table} WHERE doc IN (${rows})`, [held.key, named]);
+        }
+        connection.run(`DELETE FROM replica_entries WHERE id IN (${rows})`, [held.key, named]);
+        connection.run('DELETE FROM replica_folders WHERE key = ? AND part IN (SELECT value FROM json_each(?))', [held.key, named]);
+    }
+    const kept = [...staged].filter((part) => wanted.has(part));
+    const publishedParts = request.published === null
+        ? new Set()
+        : partsOf(connection, request.published, CONFIRMED);
+    const copied = request.parts.filter((part) => !staged.has(part) && publishedParts.has(part));
+    if (held === null && copied.length === 0) {
+        return { key: null, transfer: request.transfer, staged: [] };
+    }
+    const key = held?.key ?? newKey(connection, request.locator, request.transfer, request.fingerprint, false);
+    recomposed(connection, key, request.fingerprint);
+    if (copied.length > 0 && request.published !== null) {
+        owing(connection, key);
+        const named = JSON.stringify(copied);
+        connection.run(`INSERT INTO replica_entries (key, layer, path, visible, present, queried, content, binary, revision, file_id, created, created_instant, change, changed_instant, written, type, declared, folder, fields, links, part) SELECT ?, ${CONFIRMED}, path, 1, 1, queried, content, binary, revision, file_id, created, created_instant, change, changed_instant, written, type, declared, folder, fields, links, part FROM replica_entries WHERE key = ? AND layer = ${CONFIRMED} AND present = 1 AND part IN (SELECT value FROM json_each(?)) ORDER BY path`, [key, request.published, named]);
+        connection.run(`INSERT OR REPLACE INTO replica_folders (key, layer, path, present, part) SELECT ?, ${CONFIRMED}, path, 1, part FROM replica_folders WHERE key = ? AND layer = ${CONFIRMED} AND present = 1 AND part IN (SELECT value FROM json_each(?))`, [key, request.published, named]);
+    }
+    return { key, transfer: held?.transfer ?? request.transfer, staged: [...kept, ...copied] };
+}
 export function refreshing(connection, request) {
     const held = connection.rows('SELECT 1 FROM replica_keys k WHERE k.key = ? AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)', [request.key]);
     if (held.length === 0)
         return null;
     recomposed(connection, request.key, request.composed.fingerprint);
     const composition = request.composed.composition;
-    const owed = words();
     for (const entry of request.entries) {
-        putEntry(connection, request.key, REFRESHED, entry, true, composition, owed);
+        putEntry(connection, request.key, REFRESHED, entry, true, composition);
     }
     for (const path of request.removals) {
         putEntry(connection, request.key, REFRESHED, {
@@ -193,9 +236,11 @@ export function refreshing(connection, request) {
             binary: null,
             revision: null,
             fileId: null,
+            created: null,
             change: null,
             written: null,
-        }, false, composition, owed);
+            facts: null,
+        }, false, composition);
     }
     for (const [list, present] of [
         [request.folders, 1],
@@ -205,11 +250,9 @@ export function refreshing(connection, request) {
             connection.run('INSERT OR REPLACE INTO replica_folders (key, layer, path, present) VALUES (?, ?, ?, ?)', [request.key, REFRESHED, path, present]);
         }
     }
-    wordsWritten(connection, owed);
     return null;
 }
 function dropLayer(connection, key, layer) {
-    unindexDocuments(connection, heldIds(connection, 'SELECT id FROM replica_entries WHERE key = ? AND layer = ?', [key, layer]));
     for (const table of ['replica_values', 'replica_links']) {
         connection.run(`DELETE FROM ${table} WHERE doc IN (SELECT id FROM replica_entries WHERE key = ? AND layer = ?)`, [key, layer]);
     }
@@ -228,7 +271,6 @@ function refreshedPaths(connection, key) {
 function applyRefresh(connection, key) {
     const touched = refreshedPaths(connection, key);
     const replaced = `SELECT c.id FROM replica_entries c JOIN replica_entries r ON r.key = c.key AND r.layer = ${REFRESHED} AND r.path = c.path WHERE c.key = ? AND c.layer = ${CONFIRMED}`;
-    unindexDocuments(connection, heldIds(connection, replaced, [key]));
     for (const table of ['replica_values', 'replica_links']) {
         connection.run(`DELETE FROM ${table} WHERE doc IN (${replaced})`, [key]);
     }
@@ -254,15 +296,18 @@ function counted(connection, key, roots) {
     const [row] = connection.rows(`SELECT count(*) AS held FROM replica_entries WHERE key = ? AND visible = 1 AND present = 1 AND (binary IS NOT NULL OR substr(path, -3) = '.md') AND NOT ${beneathRoots('replica_entries.path')}`, [key, JSON.stringify(roots)]);
     return number(row.held);
 }
-export function counts(connection, rooms) {
-    return rooms.map(({ key, roots }) => counted(connection, key, roots));
-}
 function erasing(connection, key, locator) {
     connection.run('INSERT OR REPLACE INTO replica_erasing (key, locator) VALUES (?, ?)', [
         key,
         locator,
     ]);
 }
+function saved(connection, key, entry, at) {
+    if (!entry.present || entry.created !== null)
+        return entry;
+    const [held] = connection.rows(`SELECT created FROM replica_entries WHERE key = ? AND layer = ${OVERLAY} AND path = ? AND present = 1`, [key, entry.path]);
+    return { ...entry, created: text(held?.created) ?? at };
+}
 function commitRoom(connection, room) {
     const touched = new Set();
     recomposed(connection, room.key, room.composed.fingerprint);
@@ -276,51 +321,74 @@ function commitRoom(connection, room) {
         for (const path of applyRefresh(connection, room.key))
             touched.add(path);
     }
-    const owed = words();
     for (const entry of room.confirmed?.set ?? []) {
-        putEntry(connection, room.key, CONFIRMED, entry, true, composition, owed);
+        putEntry(connection, room.key, CONFIRMED, entry, true, composition);
         touched.add(entry.path);
     }
     for (const path of room.confirmed?.removed ?? []) {
-        removeEntry(connection, room.key, CONFIRMED, path, owed);
+        removeEntry(connection, room.key, CONFIRMED, path);
         touched.add(path);
     }
+    const savedAt = new Date().toISOString();
     for (const entry of room.overlay?.set ?? []) {
-        putEntry(connection, room.key, OVERLAY, entry, entry.present, composition, owed);
+        putEntry(connection, room.key, OVERLAY, saved(connection, room.key, entry, savedAt), entry.present, composition);
         touched.add(entry.path);
     }
     for (const path of room.overlay?.cleared ?? []) {
-        removeEntry(connection, room.key, OVERLAY, path, owed);
+        removeEntry(connection, room.key, OVERLAY, path);
         touched.add(path);
     }
     const changed = [...touched];
     revealed(connection, room.key, changed);
     if (room.record !== undefined) {
-        connection.run('INSERT OR REPLACE INTO replica_rooms (locator, key, record) VALUES (?, ?, ?)', [
-            room.locator,
-            room.key,
-            room.record,
-        ]);
+        const replaced = heldModel(connection, room.locator);
+        connection.run(KEEP_MODEL, [room.record, room.record, room.record, room.record]);
+        connection.run(`INSERT OR REPLACE INTO replica_rooms (locator, key, record) VALUES (?, ?, ${MODEL_LEFT})`, [room.locator, room.key, room.record, room.record, room.record]);
+        letGo(connection, replaced);
     }
-    wordsWritten(connection, owed);
     return room.published === undefined ? changed : null;
 }
+const SCHEMA = '$.model.snapshot.schema';
+const KEEP_MODEL = `INSERT OR IGNORE INTO replica_models (identity, version, schema)
+  SELECT json_extract(?, '$.model.snapshot.identity'), json_extract(?, '$.model.snapshot.version'),
+    json_extract(?, '${SCHEMA}')
+  WHERE json_type(?, '${SCHEMA}') = 'text'`;
+const MODEL_LEFT = `CASE WHEN json_type(?, '${SCHEMA}') = 'text' THEN json_set(?, '${SCHEMA}', json('null')) ELSE ? END`;
+const HELD_MODEL = `SELECT json_extract(record, '$.model.snapshot.identity') AS identity,
+    json_extract(record, '$.model.snapshot.version') AS version
+  FROM replica_rooms WHERE locator = ?`;
+const UNHELD_MODEL = `DELETE FROM replica_models WHERE identity = ? AND version = ? AND NOT EXISTS (
+  SELECT 1 FROM replica_rooms r
+  WHERE json_extract(r.record, '$.model.snapshot.identity') = replica_models.identity
+    AND json_extract(r.record, '$.model.snapshot.version') = replica_models.version
+)`;
+function heldModel(connection, locator) {
+    return connection.rows(HELD_MODEL, [locator])[0];
+}
+function letGo(connection, model) {
+    if (model !== undefined)
+        connection.run(UNHELD_MODEL, [model.identity, model.version]);
+}
+const HELD_RECORDS = `SELECT r.locator, r.key, CASE WHEN m.schema IS NULL THEN r.record
+    ELSE json_set(r.record, '${SCHEMA}', m.schema) END AS record
+  FROM replica_rooms r
+  LEFT JOIN replica_models m
+    ON m.identity = json_extract(r.record, '$.model.snapshot.identity')
+   AND m.version = json_extract(r.record, '$.model.snapshot.version')`;
 export function commit(connection, rooms) {
-    const counted = {};
     const changed = {};
     const indexed = {};
     for (const room of rooms) {
         changed[room.locator] = commitRoom(connection, room);
-        counted[room.locator] = counts(connection, [
-            { key: room.key, roots: room.roots ?? rootsOf(connection, room.locator) },
-        ])[0];
-        const [held] = connection.rows('SELECT rederiving, searching FROM replica_keys WHERE key = ?', [
+        connection.run('UPDATE replica_rooms SET documents = ? WHERE locator = ? AND key = ?', [
+            counted(connection, room.key, room.roots ?? rootsOf(connection, room.locator)),
+            room.locator,
             room.key,
         ]);
-        indexed[room.locator] =
-            held !== undefined && held.rederiving === null && held.searching === null;
+        const [held] = connection.rows('SELECT rederiving FROM replica_keys WHERE key = ?', [room.key]);
+        indexed[room.locator] = held !== undefined && held.rederiving === null;
     }
-    return { counts: counted, changed, indexed };
+    return { changed, indexed };
 }
 function rootsOf(connection, locator) {
     const [row] = connection.rows('SELECT record FROM replica_rooms WHERE locator = ?', [locator]);
@@ -332,7 +400,9 @@ export function fenceRooms(connection, locators) {
         for (const row of connection.rows('SELECT key FROM replica_keys WHERE locator = ? UNION SELECT key FROM replica_rooms WHERE locator = ?', [locator, locator])) {
             erasing(connection, number(row.key), locator);
         }
+        const replaced = heldModel(connection, locator);
         connection.run('DELETE FROM replica_rooms WHERE locator = ?', [locator]);
+        letGo(connection, replaced);
     }
     return null;
 }
@@ -357,7 +427,6 @@ export function erase(connection, locator, limit) {
         return { remaining: false };
     const key = number(held.key);
     const batch = 'SELECT id FROM replica_entries WHERE key = ? ORDER BY id LIMIT ?';
-    unindexDocuments(connection, heldIds(connection, batch, [key, limit]));
     for (const table of ['replica_values', 'replica_links']) {
         connection.run(`DELETE FROM ${table} WHERE doc IN (${batch})`, [key, limit]);
     }
@@ -377,40 +446,31 @@ function cursor(value) {
 export function rederive(connection, key, composed, bytes, ms, floor) {
     const started = performance.now();
     recomposed(connection, key, composed.fingerprint);
-    const [held] = connection.rows('SELECT rederiving, searching FROM replica_keys WHERE key = ?', [
-        key,
-    ]);
+    const [held] = connection.rows('SELECT rederiving FROM replica_keys WHERE key = ?', [key]);
     const deriving = cursor(held?.rederiving);
-    const searching = cursor(held?.searching);
-    if (deriving === null && searching === null)
+    if (deriving === null)
         return { done: true, text: 0 };
     const composition = composed.composition;
-    let last = Math.min(deriving ?? Infinity, searching ?? Infinity);
+    let last = deriving;
     let spent = 0;
     let built = 0;
     let read = false;
-    const late = () => spent >= floor && performance.now() - started >= ms;
-    const owed = [];
     reading: for (;;) {
-        const rows = connection.rows('SELECT id, path, content FROM replica_entries WHERE key = ? AND present = 1 AND id > ? ORDER BY id LIMIT ?', [key, last, DERIVED_ROWS]);
+        const rows = connection.rows('SELECT id, path, type, fields, links, octet_length(content) AS size FROM replica_entries WHERE key = ? AND present = 1 AND id > ? ORDER BY id LIMIT ?', [key, last, DERIVED_ROWS]);
         for (const row of rows) {
             const id = number(row.id);
             const path = text(row.path);
-            const content = text(row.content);
-            const owedRows = deriving !== null && id > deriving;
-            const owedWords = searching !== null && id > searching && content !== null;
-            if (owedRows) {
-                const made = derived(path, content, composition);
-                dropDerived(connection, id);
-                writeDerived(connection, id, key, made);
-            }
-            if (owedWords && described(path, content).queried)
-                owed.push({ id, path, content });
-            else if (owedRows)
-                built += content?.length ?? 0;
+            const size = row.size === null ? 0 : number(row.size);
+            dropDerived(connection, id);
+            writeDerived(connection, id, key, path, {
+                type: text(row.type),
+                fields: json(row.fields),
+                links: json(row.links) ?? [],
+            }, composition);
+            built += size;
             last = id;
-            spent += ENTRY_WEIGHT + (content?.length ?? 0);
-            if (spent >= bytes || late())
+            spent += ENTRY_WEIGHT + size;
+            if (spent >= bytes || (spent >= floor && performance.now() - started >= ms))
                 break reading;
         }
         if (rows.length < DERIVED_ROWS) {
@@ -418,23 +478,8 @@ export function rederive(connection, key, composed, bytes, ms, floor) {
             break;
         }
     }
-    let searched = last;
-    for (const [at, { id, path, content }] of owed.entries()) {
-        indexDocument(connection, id, path, content);
-        built += content.length;
-        const next = owed[at + 1];
-        if (next !== undefined && late()) {
-            searched = next.id - 1;
-            break;
-        }
-    }
-    const done = read && searched === last;
-    connection.run('UPDATE replica_keys SET rederiving = ?, searching = ? WHERE key = ?', [
-        deriving === null || read ? null : Math.max(deriving, last),
-        searching === null || done ? null : Math.max(searching, searched),
-        key,
-    ]);
-    return { done, text: built };
+    connection.run('UPDATE replica_keys SET rederiving = ? WHERE key = ?', [read ? null : last, key]);
+    return { done: read, text: built };
 }
 function heldEntry(row) {
     return {
@@ -443,12 +488,24 @@ function heldEntry(row) {
         binary: json(row.binary),
         revision: text(row.revision),
         fileId: text(row.file_id),
+        created: text(row.created),
         change: json(row.change),
         written: text(row.written),
         type: text(row.type),
+        facts: factsHeld(row),
+    };
+}
+function factsHeld(row) {
+    const type = text(row.declared);
+    if (type === null)
+        return null;
+    return {
+        type,
+        fields: json(row.fields) ?? {},
+        links: json(row.links) ?? [],
     };
 }
-const ENTRY_COLUMNS = 'e.path, e.content, e.binary, e.revision, e.file_id, e.change, e.written, e.type';
+const ENTRY_COLUMNS = 'e.path, e.content, e.binary, e.revision, e.file_id, e.created, e.change, e.written, e.type, e.declared, e.fields, e.links';
 function unrefreshed(alias) {
     return `NOT EXISTS (SELECT 1 FROM replica_entries r WHERE r.key = ${alias}.key AND r.layer = ${REFRESHED} AND r.path = ${alias}.path)`;
 }
@@ -459,22 +516,6 @@ function viewed(view, alias) {
         return `${alias}.layer = ${CONFIRMED}`;
     return `${alias}.present = 1 AND (${alias}.layer = ${REFRESHED} OR (${alias}.layer = ${CONFIRMED} AND ${unrefreshed(alias)}))`;
 }
-function layers(view) {
-    if (view !== 'refreshed')
-        return [viewed(view, 'e')];
-    return [
-        `e.layer = ${REFRESHED} AND e.present = 1`,
-        `e.layer = ${CONFIRMED} AND e.present = 1 AND ${unrefreshed('e')}`,
-    ];
-}
-function pathOrdered(connection, columns, key, view, condition, after, limit) {
-    const beyond = after === null ? '' : ' AND e.path > ?';
-    const parts = layers(view).map((layer) => `SELECT ${columns} FROM replica_entries e WHERE e.key = ? AND ${layer}${condition}${beyond}`);
-    return connection.rows(`${parts.join(' UNION ALL ')} ORDER BY path LIMIT ?`, [
-        ...parts.flatMap(() => (after === null ? [key] : [key, after])),
-        limit,
-    ]);
-}
 export function entries(connection, key, view, list) {
     if (list.length === 0)
         return [];
@@ -493,37 +534,12 @@ export function typed(connection, key, view, list) {
         binary: number(row.binary) === 1,
     }));
 }
-export function referrers(connection, key, view, targets, kinds) {
+export function referrers(connection, key, view, targets) {
     if (targets.length === 0)
         return [];
     return connection
-        .rows(`SELECT DISTINCT e.path, l.target, l.kind FROM replica_links l JOIN replica_entries e ON e.id = l.doc WHERE l.key = ? AND l.target IN (SELECT value FROM json_each(?)) AND l.kind IN (SELECT value FROM json_each(?)) AND ${viewed(view, 'e')}`, [key, paths(targets), JSON.stringify(kinds)])
-        .map((row) => ({
-        path: text(row.path),
-        target: text(row.target),
-        kind: text(row.kind),
-    }));
-}
-export function walk(connection, key, view, after, bytes) {
-    const found = [];
-    let spent = 0;
-    let from = after;
-    for (;;) {
-        const rows = pathOrdered(connection, ENTRY_COLUMNS, key, view, '', from, 64);
-        for (const row of rows) {
-            const entry = heldEntry(row);
-            found.push(entry);
-            spent += 256 + (entry.content?.length ?? 0) * 2;
-            if (spent >= bytes)
-                return { entries: found, next: entry.path };
-        }
-        if (rows.length < 64)
-            return { entries: found, next: null };
-        from = found.at(-1).path;
-    }
-}
-export function files(connection, key, view, after, limit) {
-    return pathOrdered(connection, 'e.path, e.revision', key, view, ' AND e.content IS NOT NULL AND e.revision IS NOT NULL', after, limit).map((row) => ({ path: text(row.path), revision: text(row.revision) }));
+        .rows(`SELECT DISTINCT e.path, l.target FROM replica_links l JOIN replica_entries e ON e.id = l.doc WHERE l.key = ? AND l.target IN (SELECT value FROM json_each(?)) AND ${viewed(view, 'e')}`, [key, paths(targets)])
+        .map((row) => ({ path: text(row.path), target: text(row.target) }));
 }
 export function folders(connection, key, view, list) {
     return list.filter((path) => connection.rows(`SELECT 1 FROM replica_folders f WHERE f.key = ? AND f.layer = ${CONFIRMED} AND f.present = 1 AND f.path = ? UNION ALL SELECT 1 FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.path >= ? AND e.path < ? LIMIT 1`, [key, path, key, `${path}/`, `${path}0`]).length > 0);
@@ -531,50 +547,103 @@ export function folders(connection, key, view, list) {
 export function typedWithin(connection, key, view, roots, within) {
     return (connection.rows(`SELECT 1 FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.declared IS NOT NULL AND e.declared <> 'any' AND ${within ? '' : 'NOT '}${beneathRoots('e.path')} LIMIT 1`, [key, JSON.stringify(roots)]).length > 0);
 }
+const ORPHANS = `SELECT k.key FROM replica_keys k
+  WHERE NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
+    AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)
+    AND (${UNPARTED} OR NOT EXISTS (
+      SELECT 1 FROM replica_entries e WHERE e.key = k.key
+      UNION ALL SELECT 1 FROM replica_folders f WHERE f.key = k.key
+    ))`;
+const STAGED = `SELECT DISTINCT k.locator FROM replica_keys k
+  WHERE NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
+    AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)
+    AND k.key NOT IN (${ORPHANS})`;
 export function opened(connection) {
-    const facts = {};
-    for (const row of connection.rows('SELECT name, value FROM replica_facts')) {
-        facts[text(row.name)] = text(row.value);
-    }
     return {
-        rooms: connection.rows('SELECT locator, key, record FROM replica_rooms').map((row) => ({
-            locator: text(row.locator),
-            key: number(row.key),
-            record: text(row.record),
-        })),
-        keys: connection
-            .rows('SELECT key, locator, transfer, rederiving, searching FROM replica_keys')
-            .map((row) => ({
-            key: number(row.key),
-            locator: text(row.locator),
-            transfer: text(row.transfer),
-            rederiving: cursor(row.rederiving),
-            searching: cursor(row.searching),
-        })),
-        facts,
-        presentation: connection.rows('SELECT app, value FROM replica_presentation').map((row) => ({
-            app: text(row.app),
-            value: text(row.value),
-        })),
-        overlaid: connection
-            .rows(`SELECT key, path FROM replica_entries WHERE layer = ${OVERLAY}`)
-            .map((row) => ({ key: number(row.key), path: text(row.path) })),
-        orphans: connection
-            .rows(`SELECT k.key FROM replica_keys k
-         WHERE NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
-           AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)`)
-            .map((row) => number(row.key)),
-        erasing: connection.rows('SELECT key, locator FROM replica_erasing').map((row) => ({
-            key: number(row.key),
-            locator: text(row.locator),
-        })),
+        orphans: connection.rows(ORPHANS).map((row) => number(row.key)),
+        erasing: connection.rows('SELECT 1 FROM replica_erasing LIMIT 1').length > 0,
+    };
+}
+const ROOM_ROWS = `SELECT h.locator, h.key, h.record, r.documents, k.rederiving IS NULL AS indexed
+  FROM (${HELD_RECORDS}) h
+  JOIN replica_rooms r ON r.locator = h.locator
+  LEFT JOIN replica_keys k ON k.key = h.key`;
+function roomRow(row) {
+    return {
+        locator: text(row.locator),
+        key: number(row.key),
+        record: text(row.record),
+        indexed: number(row.indexed) === 1,
+        documents: number(row.documents),
     };
 }
+export function heldRooms(connection, locators) {
+    return (locators === null
+        ? connection.rows(`${ROOM_ROWS} ORDER BY h.locator`)
+        : connection.rows(`${ROOM_ROWS} WHERE h.locator IN (SELECT value FROM json_each(?)) ORDER BY h.locator`, [JSON.stringify(locators)])).map(roomRow);
+}
+export function heldRoom(connection, locator) {
+    return heldRooms(connection, [locator])[0] ?? null;
+}
+export function overlaid(connection, key) {
+    return connection
+        .rows(`SELECT path FROM replica_entries WHERE key = ? AND layer = ${OVERLAY}`, [key])
+        .map((row) => text(row.path));
+}
+export function covering(connection) {
+    return connection
+        .rows(`SELECT r.locator, r.key, r.documents, json_extract(r.record, '$.transfer') AS transfer,
+         json_extract(r.record, '$.roomVersion') AS room_version,
+         coalesce(json_extract(r.record, '$.inventory.bytes'), 0) AS bytes,
+         json_extract(r.record, '$.replica.offline') AS offline,
+         k.rederiving IS NULL AS indexed
+       FROM replica_rooms r LEFT JOIN replica_keys k ON k.key = r.key ORDER BY r.locator`)
+        .map((row) => ({
+        locator: text(row.locator),
+        key: number(row.key),
+        transfer: text(row.transfer),
+        roomVersion: text(row.room_version),
+        indexed: number(row.indexed) === 1,
+        documents: number(row.documents),
+        bytes: number(row.bytes),
+        offline: text(row.offline),
+    }));
+}
+export function offlineMode(connection, locator) {
+    const [held] = connection.rows("SELECT json_extract(record, '$.replica.offline') AS offline FROM replica_rooms WHERE locator = ?", [locator]);
+    return text(held?.offline);
+}
+export function stagedLocators(connection) {
+    return connection.rows(STAGED).map((row) => text(row.locator));
+}
+export function heldLocators(connection) {
+    return connection
+        .rows(`SELECT locator FROM replica_rooms
+       UNION SELECT locator FROM authored_room_drafts
+       UNION SELECT locator FROM authored_queues
+       UNION ${STAGED}`)
+        .map((row) => text(row.locator));
+}
+export function demoted(connection) {
+    connection.run(`UPDATE replica_rooms SET record = json_set(record, '$.model.status', 'cached')
+     WHERE json_extract(record, '$.model.status') = 'current'`);
+    return null;
+}
+export function facts(connection) {
+    const held = {};
+    for (const row of connection.rows('SELECT name, value FROM replica_facts')) {
+        held[text(row.name)] = text(row.value);
+    }
+    return held;
+}
+export function presentation(connection, app) {
+    const [held] = connection.rows('SELECT value FROM replica_presentation WHERE app = ?', [app]);
+    return text(held?.value);
+}
 export function ended(connection) {
     for (const statement of REPLICA_TABLES) {
         const table = /^CREATE TABLE (\w+)/u.exec(statement)?.[1];
         if (table !== undefined)
             connection.run(`DELETE FROM ${table}`);
     }
-    unindexAll(connection);
 }