@agentshouse/mdruntime 0.1.0-ccbb89b0ca151 → 0.1.0-ccc0e3407f107

package/dist/stored.js

@@ -1,23 +1,21 @@
-import { indexDocument, unindexAll, unindexDocuments } from './fulltext.js';
-import { described, derived } from './rows.js';
+import { sql } from '@agentshouse/mdmodel';
+import { composedOf } from './collection.js';
+import { described, derived, heldOf } from './rows.js';
 export const REPLICA_TABLES = [
-    'CREATE TABLE replica_keys (key INTEGER PRIMARY KEY AUTOINCREMENT, locator TEXT NOT NULL, transfer TEXT NOT NULL, composition TEXT NOT NULL, rederiving INTEGER, searching INTEGER)',
+    'CREATE TABLE replica_keys (key INTEGER PRIMARY KEY AUTOINCREMENT, locator TEXT NOT NULL, transfer TEXT NOT NULL, fingerprint TEXT NOT NULL, rederiving INTEGER)',
     'CREATE TABLE replica_rooms (locator TEXT PRIMARY KEY, key INTEGER NOT NULL, record TEXT NOT NULL) WITHOUT ROWID',
-    'CREATE TABLE replica_transfers (locator TEXT PRIMARY KEY, key INTEGER NOT NULL, record TEXT NOT NULL) WITHOUT ROWID',
-    'CREATE TABLE replica_transfer_parts (locator TEXT NOT NULL, token TEXT NOT NULL, part BLOB NOT NULL)',
-    'CREATE INDEX replica_transfer_parts_locator ON replica_transfer_parts (locator)',
-    'CREATE TABLE replica_entries (id INTEGER PRIMARY KEY, key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, visible INTEGER NOT NULL, present INTEGER NOT NULL, queried INTEGER NOT NULL, binary TEXT, revision TEXT, file_id TEXT, change TEXT, written TEXT, type TEXT, declared TEXT, folder TEXT NOT NULL, fields TEXT, content TEXT)',
+    'CREATE TABLE replica_models (identity TEXT NOT NULL, version TEXT NOT NULL, schema TEXT NOT NULL, PRIMARY KEY (identity, version)) WITHOUT ROWID',
+    'CREATE TABLE replica_entries (id INTEGER PRIMARY KEY, key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, visible INTEGER NOT NULL, present INTEGER NOT NULL, queried INTEGER NOT NULL, binary TEXT, revision TEXT, file_id TEXT, change TEXT, written TEXT, type TEXT, declared TEXT, folder TEXT NOT NULL, fields TEXT, links TEXT, content TEXT)',
     'CREATE UNIQUE INDEX replica_entries_path ON replica_entries (key, layer, path)',
-    'CREATE INDEX replica_entries_visible ON replica_entries (key, visible, path)',
+    'CREATE INDEX replica_entries_visible ON replica_entries (key, visible, present, path, binary)',
     'CREATE INDEX replica_entries_folder ON replica_entries (key, folder, path)',
-    'CREATE TABLE replica_values (doc INTEGER NOT NULL, key INTEGER NOT NULL, keys TEXT NOT NULL, places TEXT NOT NULL, item INTEGER NOT NULL, kind TEXT NOT NULL, value, size INTEGER, PRIMARY KEY (doc, keys, places, item)) WITHOUT ROWID',
-    'CREATE INDEX replica_values_lookup ON replica_values (key, keys, kind, value)',
+    'CREATE INDEX replica_entries_type ON replica_entries (key, type, visible, present, queried, folder)',
+    'CREATE INDEX replica_entries_key ON replica_entries (key)',
+    sql.VALUE_TABLE,
     'CREATE TABLE replica_links (doc INTEGER NOT NULL, key INTEGER NOT NULL, kind TEXT NOT NULL, keys TEXT NOT NULL, places TEXT NOT NULL, target TEXT NOT NULL, field TEXT NOT NULL, PRIMARY KEY (doc, kind, keys, places, target)) WITHOUT ROWID',
     'CREATE INDEX replica_links_target ON replica_links (key, target, kind, keys)',
     'CREATE TABLE replica_folders (key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, present INTEGER NOT NULL, PRIMARY KEY (key, layer, path)) WITHOUT ROWID',
     'CREATE TABLE replica_erasing (key INTEGER PRIMARY KEY, locator TEXT NOT NULL)',
-    'CREATE TABLE replica_received (key TEXT PRIMARY KEY, head TEXT NOT NULL) WITHOUT ROWID',
-    'CREATE TABLE replica_received_parts (key TEXT NOT NULL, part INTEGER NOT NULL, value TEXT NOT NULL, PRIMARY KEY (key, part)) WITHOUT ROWID',
     'CREATE TABLE replica_facts (name TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID',
     'CREATE TABLE replica_presentation (app TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID',
 ];
@@ -32,10 +30,8 @@ const ABSENT = {
     queried: false,
     folder: '',
     fields: null,
-    values: [],
-    links: [],
+    links: null,
 };
-const compositions = new WeakMap();
 function text(value) {
     return typeof value === 'string' ? value : null;
 }
@@ -51,44 +47,22 @@ function flag(value) {
 function paths(list) {
     return JSON.stringify(list);
 }
-function compositionOf(connection, key) {
-    let held = compositions.get(connection);
-    if (held === undefined) {
-        held = new Map();
-        compositions.set(connection, held);
-    }
-    const [row] = connection.rows('SELECT composition FROM replica_keys WHERE key = ?', [key]);
-    const textual = text(row?.composition) ?? 'null';
-    const known = held.get(textual);
-    if (known !== undefined)
-        return known;
-    const parsed = JSON.parse(textual) ?? {
-        model: { status: 'unavailable', reason: 'not-acquired' },
-        selection: null,
-    };
-    held.clear();
-    held.set(textual, parsed);
-    return parsed;
-}
-export function fingerprint(composition) {
-    const shape = (model) => model.status === 'unavailable' ? null : model.types;
-    return JSON.stringify({
-        model: shape(composition.model),
-        selection: composition.selection === null
-            ? null
-            : { roots: composition.selection.roots, model: shape(composition.selection.model) },
-    });
-}
 function dropDerived(connection, doc) {
     connection.run('DELETE FROM replica_values WHERE doc = ?', [doc]);
     connection.run('DELETE FROM replica_links WHERE doc = ?', [doc]);
 }
-function writeDerived(connection, doc, key, made) {
-    for (const value of made.values) {
-        connection.run('INSERT INTO replica_values (doc, key, keys, places, item, kind, value, size) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', [doc, key, value.keys, value.places, value.item, value.kind, value.value, value.size]);
+function writeDerived(connection, doc, key, path, held, composition) {
+    const made = derived(path, held, composition);
+    if (made.values.length > 0) {
+        const inserted = sql.insertedValues(doc, made.values);
+        connection.run(inserted.sql, inserted.bind);
     }
-    for (const link of made.links) {
-        connection.run('INSERT OR IGNORE INTO replica_links (doc, key, kind, keys, places, target, field) VALUES (?, ?, ?, ?, ?, ?, ?)', [doc, key, link.kind, link.keys, link.places, link.target, link.field]);
+    if (made.links.length > 0) {
+        connection.run('INSERT OR IGNORE INTO replica_links (doc, key, kind, keys, places, target, field) SELECT ?, ?, value ->> 0, value ->> 1, value ->> 2, value ->> 3, value ->> 4 FROM json_each(?)', [
+            doc,
+            key,
+            JSON.stringify(made.links.map((link) => [link.kind, link.keys, link.places, link.target, link.field])),
+        ]);
     }
 }
 function entryId(connection, key, layer, path) {
@@ -100,14 +74,10 @@ function removeEntry(connection, key, layer, path) {
     if (id === null)
         return;
     dropDerived(connection, id);
-    unindexDocuments(connection, [id]);
     connection.run('DELETE FROM replica_entries WHERE id = ?', [id]);
 }
-function heldIds(connection, sql, bind) {
-    return connection.rows(sql, bind).map((row) => number(row.id));
-}
 function insertEntry(connection, key, layer, entry, present, made) {
-    connection.run('INSERT INTO replica_entries (key, layer, path, visible, present, queried, content, binary, revision, file_id, change, written, type, declared, folder, fields) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [
+    connection.run('INSERT INTO replica_entries (key, layer, path, visible, present, queried, content, binary, revision, file_id, change, written, type, declared, folder, fields, links) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [
         key,
         layer,
         entry.path,
@@ -124,82 +94,54 @@ function insertEntry(connection, key, layer, entry, present, made) {
         made.declared,
         made.folder,
         made.fields,
+        made.links,
     ]);
 }
 function textOf(entry) {
     return entry.binary === null ? entry.content : null;
 }
-function putEntry(connection, key, layer, entry, present, composition, searched = true) {
+function putEntry(connection, key, layer, entry, present, composition) {
     removeEntry(connection, key, layer, entry.path);
     const content = textOf(entry);
-    const made = present ? derived(entry.path, content, composition) : ABSENT;
-    insertEntry(connection, key, layer, entry, present, made);
+    insertEntry(connection, key, layer, entry, present, present ? described(entry.path, content, entry.facts) : ABSENT);
     if (!present)
         return;
     const id = entryId(connection, key, layer, entry.path);
-    writeDerived(connection, id, key, made);
-    if (searched && content !== null && made.queried) {
-        indexDocument(connection, id, entry.path, content);
-    }
+    writeDerived(connection, id, key, entry.path, heldOf(entry.path, content, entry.facts), composition);
 }
 function placeEntry(connection, key, entry) {
     removeEntry(connection, key, CONFIRMED, entry.path);
-    insertEntry(connection, key, CONFIRMED, entry, true, described(entry.path, textOf(entry)));
+    insertEntry(connection, key, CONFIRMED, entry, true, described(entry.path, textOf(entry), entry.facts));
 }
 function revealed(connection, key, touched) {
     if (touched.length === 0)
         return;
     connection.run(`UPDATE replica_entries SET visible = CASE WHEN layer = ${OVERLAY} THEN present ELSE NOT EXISTS (SELECT 1 FROM replica_entries o WHERE o.key = replica_entries.key AND o.layer = ${OVERLAY} AND o.path = replica_entries.path) END WHERE key = ? AND layer IN (${CONFIRMED}, ${OVERLAY}) AND path IN (SELECT value FROM json_each(?))`, [key, paths(touched)]);
 }
-function newKey(connection, locator, transfer, composition, built) {
+function newKey(connection, locator, transfer, fingerprint, built) {
     const owed = built ? null : 0;
-    connection.run('INSERT INTO replica_keys (locator, transfer, composition, rederiving, searching) VALUES (?, ?, ?, ?, ?)', [locator, transfer, JSON.stringify(composition), owed, owed]);
+    connection.run('INSERT INTO replica_keys (locator, transfer, fingerprint, rederiving) VALUES (?, ?, ?, ?)', [locator, transfer, fingerprint, owed]);
     const [row] = connection.rows('SELECT last_insert_rowid() AS key');
     return number(row.key);
 }
-function recomposed(connection, key, composition) {
-    const [row] = connection.rows('SELECT composition FROM replica_keys WHERE key = ?', [key]);
-    const before = json(row?.composition);
-    const held = connection.rows('SELECT 1 FROM replica_entries WHERE key = ? LIMIT 1', [key]);
-    if (held.length === 0 || (before !== null && fingerprint(before) === fingerprint(composition))) {
-        connection.run('UPDATE replica_keys SET composition = ? WHERE key = ?', [
-            JSON.stringify(composition),
-            key,
-        ]);
+function recomposed(connection, key, fingerprint) {
+    const [row] = connection.rows('SELECT fingerprint FROM replica_keys WHERE key = ?', [key]);
+    if (row === undefined || text(row.fingerprint) === fingerprint)
         return;
-    }
-    connection.run('UPDATE replica_keys SET composition = ?, rederiving = 0 WHERE key = ?', [
-        JSON.stringify(composition),
-        key,
-    ]);
+    const held = connection.rows('SELECT 1 FROM replica_entries WHERE key = ? LIMIT 1', [key]);
+    connection.run(`UPDATE replica_keys SET fingerprint = ?${held.length === 0 ? '' : ', rederiving = 0'} WHERE key = ?`, [fingerprint, key]);
 }
 export function entered(connection, locator, composition, rows) {
-    const key = newKey(connection, locator, '', composition, true);
+    const key = newKey(connection, locator, '', composedOf(composition).fingerprint, true);
     for (const entry of rows)
-        putEntry(connection, key, CONFIRMED, entry, true, composition, false);
+        putEntry(connection, key, CONFIRMED, entry, true, composition);
     return key;
 }
-function dropTransfer(connection, locator) {
-    connection.run('DELETE FROM replica_transfers WHERE locator = ?', [locator]);
-    connection.run('DELETE FROM replica_transfer_parts WHERE locator = ?', [locator]);
-}
 export function stage(connection, request) {
-    const [held] = connection.rows('SELECT key FROM replica_transfers WHERE locator = ?', [
-        request.locator,
-    ]);
     const key = request.key ??
-        (held === undefined
-            ? newKey(connection, request.locator, request.transfer, request.composition, false)
-            : number(held.key));
-    recomposed(connection, key, request.composition);
-    connection.run('UPDATE replica_keys SET rederiving = coalesce(rederiving, last), searching = coalesce(searching, last) FROM (SELECT coalesce(max(id), 0) AS last FROM replica_entries) WHERE key = ?', [key]);
-    connection.run('INSERT OR REPLACE INTO replica_transfers (locator, key, record) VALUES (?, ?, ?)', [request.locator, key, request.record]);
-    if (request.written !== undefined) {
-        connection.run('DELETE FROM replica_transfer_parts WHERE locator = ? AND token = ?', [
-            request.locator,
-            request.written,
-        ]);
-    }
+        newKey(connection, request.locator, request.transfer, request.fingerprint, false);
+    recomposed(connection, key, request.fingerprint);
+    connection.run('UPDATE replica_keys SET rederiving = coalesce(rederiving, last) FROM (SELECT coalesce(max(id), 0) AS last FROM replica_entries) WHERE key = ?', [key]);
     for (const entry of request.entries)
         placeEntry(connection, key, entry);
     for (const path of request.folders) {
@@ -208,10 +150,11 @@ export function stage(connection, request) {
     return key;
 }
 export function refreshing(connection, request) {
-    const held = connection.rows('SELECT 1 FROM replica_rooms WHERE key = ? UNION ALL SELECT 1 FROM replica_transfers WHERE key = ?', [request.key, request.key]);
+    const held = connection.rows('SELECT 1 FROM replica_keys k WHERE k.key = ? AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)', [request.key]);
     if (held.length === 0)
         return null;
-    const composition = compositionOf(connection, request.key);
+    recomposed(connection, request.key, request.composed.fingerprint);
+    const composition = request.composed.composition;
     for (const entry of request.entries) {
         putEntry(connection, request.key, REFRESHED, entry, true, composition);
     }
@@ -224,6 +167,7 @@ export function refreshing(connection, request) {
             fileId: null,
             change: null,
             written: null,
+            facts: null,
         }, false, composition);
     }
     for (const [list, present] of [
@@ -237,7 +181,6 @@ export function refreshing(connection, request) {
     return null;
 }
 function dropLayer(connection, key, layer) {
-    unindexDocuments(connection, heldIds(connection, 'SELECT id FROM replica_entries WHERE key = ? AND layer = ?', [key, layer]));
     for (const table of ['replica_values', 'replica_links']) {
         connection.run(`DELETE FROM ${table} WHERE doc IN (SELECT id FROM replica_entries WHERE key = ? AND layer = ?)`, [key, layer]);
     }
@@ -256,7 +199,6 @@ function refreshedPaths(connection, key) {
 function applyRefresh(connection, key) {
     const touched = refreshedPaths(connection, key);
     const replaced = `SELECT c.id FROM replica_entries c JOIN replica_entries r ON r.key = c.key AND r.layer = ${REFRESHED} AND r.path = c.path WHERE c.key = ? AND c.layer = ${CONFIRMED}`;
-    unindexDocuments(connection, heldIds(connection, replaced, [key]));
     for (const table of ['replica_values', 'replica_links']) {
         connection.run(`DELETE FROM ${table} WHERE doc IN (${replaced})`, [key]);
     }
@@ -275,12 +217,11 @@ function applyRefresh(connection, key) {
     connection.run('DELETE FROM replica_folders WHERE key = ? AND layer = ?', [key, REFRESHED]);
     return touched;
 }
+function beneathRoots(path) {
+    return `EXISTS (SELECT 1 FROM json_each(?) r WHERE ${path} = r.value OR (${path} >= r.value || '/' AND ${path} < r.value || '0'))`;
+}
 function counted(connection, key, roots) {
-    const outside = roots.map(() => '(path = ? OR (path >= ? AND path < ?))').join(' OR ');
-    const bind = [key];
-    for (const root of roots)
-        bind.push(root, `${root}/`, `${root}0`);
-    const [row] = connection.rows(`SELECT count(*) AS held FROM replica_entries WHERE key = ? AND visible = 1 AND present = 1 AND (binary IS NOT NULL OR substr(path, -3) = '.md')${roots.length === 0 ? '' : ` AND NOT (${outside})`}`, bind);
+    const [row] = connection.rows(`SELECT count(*) AS held FROM replica_entries WHERE key = ? AND visible = 1 AND present = 1 AND (binary IS NOT NULL OR substr(path, -3) = '.md') AND NOT ${beneathRoots('replica_entries.path')}`, [key, JSON.stringify(roots)]);
     return number(row.held);
 }
 export function counts(connection, rooms) {
@@ -294,14 +235,12 @@ function erasing(connection, key, locator) {
 }
 function commitRoom(connection, room) {
     const touched = new Set();
-    if (room.composition !== undefined)
-        recomposed(connection, room.key, room.composition);
-    const composition = compositionOf(connection, room.key);
-    if (room.published !== undefined) {
-        if (room.published.from !== null && room.published.from !== room.key) {
-            erasing(connection, room.published.from, room.locator);
-        }
-        dropTransfer(connection, room.locator);
+    recomposed(connection, room.key, room.composed.fingerprint);
+    const composition = room.composed.composition;
+    if (room.published !== undefined &&
+        room.published.from !== null &&
+        room.published.from !== room.key) {
+        erasing(connection, room.published.from, room.locator);
     }
     if (room.refreshed === true) {
         for (const path of applyRefresh(connection, room.key))
@@ -326,14 +265,28 @@ function commitRoom(connection, room) {
     const changed = [...touched];
     revealed(connection, room.key, changed);
     if (room.record !== undefined) {
-        connection.run('INSERT OR REPLACE INTO replica_rooms (locator, key, record) VALUES (?, ?, ?)', [
-            room.locator,
-            room.key,
-            room.record,
-        ]);
+        connection.run(KEEP_MODEL, [room.record, room.record, room.record, room.record]);
+        connection.run(`INSERT OR REPLACE INTO replica_rooms (locator, key, record) VALUES (?, ?, ${MODEL_LEFT})`, [room.locator, room.key, room.record, room.record, room.record]);
     }
     return room.published === undefined ? changed : null;
 }
+const SCHEMA = '$.model.snapshot.schema';
+const KEEP_MODEL = `INSERT OR IGNORE INTO replica_models (identity, version, schema)
+  SELECT json_extract(?, '$.model.snapshot.identity'), json_extract(?, '$.model.snapshot.version'),
+    json_extract(?, '${SCHEMA}')
+  WHERE json_type(?, '${SCHEMA}') = 'text'`;
+const MODEL_LEFT = `CASE WHEN json_type(?, '${SCHEMA}') = 'text' THEN json_set(?, '${SCHEMA}', json('null')) ELSE ? END`;
+const UNHELD_MODELS = `DELETE FROM replica_models WHERE NOT EXISTS (
+  SELECT 1 FROM replica_rooms r
+  WHERE json_extract(r.record, '$.model.snapshot.identity') = replica_models.identity
+    AND json_extract(r.record, '$.model.snapshot.version') = replica_models.version
+)`;
+const HELD_RECORDS = `SELECT r.locator, r.key, CASE WHEN m.schema IS NULL THEN r.record
+    ELSE json_set(r.record, '${SCHEMA}', m.schema) END AS record
+  FROM replica_rooms r
+  LEFT JOIN replica_models m
+    ON m.identity = json_extract(r.record, '$.model.snapshot.identity')
+   AND m.version = json_extract(r.record, '$.model.snapshot.version')`;
 export function commit(connection, rooms) {
     const counted = {};
     const changed = {};
@@ -343,12 +296,11 @@ export function commit(connection, rooms) {
         counted[room.locator] = counts(connection, [
             { key: room.key, roots: room.roots ?? rootsOf(connection, room.locator) },
         ])[0];
-        const [held] = connection.rows('SELECT rederiving, searching FROM replica_keys WHERE key = ?', [
-            room.key,
-        ]);
-        indexed[room.locator] =
-            held !== undefined && held.rederiving === null && held.searching === null;
+        const [held] = connection.rows('SELECT rederiving FROM replica_keys WHERE key = ?', [room.key]);
+        indexed[room.locator] = held !== undefined && held.rederiving === null;
     }
+    if (rooms.some((room) => room.record !== undefined))
+        connection.run(UNHELD_MODELS);
     return { counts: counted, changed, indexed };
 }
 function rootsOf(connection, locator) {
@@ -358,27 +310,25 @@ function rootsOf(connection, locator) {
 }
 export function fenceRooms(connection, locators) {
     for (const locator of locators) {
-        for (const row of connection.rows('SELECT key FROM replica_keys WHERE locator = ? UNION SELECT key FROM replica_rooms WHERE locator = ? UNION SELECT key FROM replica_transfers WHERE locator = ?', [locator, locator, locator])) {
+        for (const row of connection.rows('SELECT key FROM replica_keys WHERE locator = ? UNION SELECT key FROM replica_rooms WHERE locator = ?', [locator, locator])) {
             erasing(connection, number(row.key), locator);
         }
         connection.run('DELETE FROM replica_rooms WHERE locator = ?', [locator]);
-        dropTransfer(connection, locator);
     }
+    connection.run(UNHELD_MODELS);
     return null;
 }
-export function abandon(connection, locator, kept) {
-    const [held] = connection.rows('SELECT key FROM replica_transfers WHERE locator = ?', [locator]);
-    dropTransfer(connection, locator);
-    if (held === undefined)
-        return null;
-    if (!kept)
-        erasing(connection, number(held.key), locator);
-    return number(held.key);
+export function abandon(connection, locator) {
+    for (const row of connection.rows(`SELECT k.key FROM replica_keys k WHERE k.locator = ?
+       AND NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
+       AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)`, [locator])) {
+        erasing(connection, number(row.key), locator);
+    }
+    return null;
 }
 export function release(connection, key) {
     const [held] = connection.rows(`SELECT k.locator FROM replica_keys k WHERE k.key = ?
-       AND NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
-       AND NOT EXISTS (SELECT 1 FROM replica_transfers t WHERE t.key = k.key)`, [key]);
+       AND NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)`, [key]);
     if (held !== undefined)
         erasing(connection, key, text(held.locator));
     return null;
@@ -389,7 +339,6 @@ export function erase(connection, locator, limit) {
         return { remaining: false };
     const key = number(held.key);
     const batch = 'SELECT id FROM replica_entries WHERE key = ? ORDER BY id LIMIT ?';
-    unindexDocuments(connection, heldIds(connection, batch, [key, limit]));
     for (const table of ['replica_values', 'replica_links']) {
         connection.run(`DELETE FROM ${table} WHERE doc IN (${batch})`, [key, limit]);
     }
@@ -406,50 +355,43 @@ export function erase(connection, locator, limit) {
 function cursor(value) {
     return value === null || value === undefined ? null : number(value);
 }
-export function rederive(connection, key, bytes) {
-    const [held] = connection.rows('SELECT rederiving, searching FROM replica_keys WHERE key = ?', [
-        key,
-    ]);
+export function rederive(connection, key, composed, bytes, ms, floor) {
+    const started = performance.now();
+    recomposed(connection, key, composed.fingerprint);
+    const [held] = connection.rows('SELECT rederiving FROM replica_keys WHERE key = ?', [key]);
     const deriving = cursor(held?.rederiving);
-    const searching = cursor(held?.searching);
-    if (deriving === null && searching === null)
-        return { done: true };
-    const composition = compositionOf(connection, key);
-    let last = Math.min(deriving ?? Infinity, searching ?? Infinity);
+    if (deriving === null)
+        return { done: true, text: 0 };
+    const composition = composed.composition;
+    let last = deriving;
     let spent = 0;
-    const reached = (done) => {
-        const next = (from) => (from === null || done ? null : Math.max(from, last));
-        connection.run('UPDATE replica_keys SET rederiving = ?, searching = ? WHERE key = ?', [
-            next(deriving),
-            next(searching),
-            key,
-        ]);
-        return { done };
-    };
-    for (;;) {
-        const rows = connection.rows('SELECT id, path, content FROM replica_entries WHERE key = ? AND present = 1 AND id > ? ORDER BY id LIMIT ?', [key, last, DERIVED_ROWS]);
+    let built = 0;
+    let read = false;
+    reading: for (;;) {
+        const rows = connection.rows('SELECT id, path, type, fields, links, octet_length(content) AS size FROM replica_entries WHERE key = ? AND present = 1 AND id > ? ORDER BY id LIMIT ?', [key, last, DERIVED_ROWS]);
         for (const row of rows) {
             const id = number(row.id);
             const path = text(row.path);
-            const content = text(row.content);
-            const owedRows = deriving !== null && id > deriving;
-            const owedWords = searching !== null && id > searching && content !== null;
-            if (owedRows) {
-                const made = derived(path, content, composition);
-                dropDerived(connection, id);
-                writeDerived(connection, id, key, made);
-            }
-            if (owedWords && described(path, content).queried) {
-                indexDocument(connection, id, path, content);
-            }
+            const size = row.size === null ? 0 : number(row.size);
+            dropDerived(connection, id);
+            writeDerived(connection, id, key, path, {
+                type: text(row.type),
+                fields: json(row.fields),
+                links: json(row.links) ?? [],
+            }, composition);
+            built += size;
             last = id;
-            spent += ENTRY_WEIGHT + (content?.length ?? 0);
-            if (spent >= bytes)
-                return reached(false);
+            spent += ENTRY_WEIGHT + size;
+            if (spent >= bytes || (spent >= floor && performance.now() - started >= ms))
+                break reading;
+        }
+        if (rows.length < DERIVED_ROWS) {
+            read = true;
+            break;
         }
-        if (rows.length < DERIVED_ROWS)
-            return reached(true);
     }
+    connection.run('UPDATE replica_keys SET rederiving = ? WHERE key = ?', [read ? null : last, key]);
+    return { done: read, text: built };
 }
 function heldEntry(row) {
     return {
@@ -461,15 +403,45 @@ function heldEntry(row) {
         change: json(row.change),
         written: text(row.written),
         type: text(row.type),
+        facts: factsHeld(row),
+    };
+}
+function factsHeld(row) {
+    const type = text(row.declared);
+    if (type === null)
+        return null;
+    return {
+        type,
+        fields: json(row.fields) ?? {},
+        links: json(row.links) ?? [],
     };
 }
-const ENTRY_COLUMNS = 'e.path, e.content, e.binary, e.revision, e.file_id, e.change, e.written, e.type';
+const ENTRY_COLUMNS = 'e.path, e.content, e.binary, e.revision, e.file_id, e.change, e.written, e.type, e.declared, e.fields, e.links';
+function unrefreshed(alias) {
+    return `NOT EXISTS (SELECT 1 FROM replica_entries r WHERE r.key = ${alias}.key AND r.layer = ${REFRESHED} AND r.path = ${alias}.path)`;
+}
 function viewed(view, alias) {
     if (view === 'visible')
         return `${alias}.visible = 1 AND ${alias}.present = 1`;
     if (view === 'confirmed')
         return `${alias}.layer = ${CONFIRMED}`;
-    return `${alias}.present = 1 AND (${alias}.layer = ${REFRESHED} OR (${alias}.layer = ${CONFIRMED} AND NOT EXISTS (SELECT 1 FROM replica_entries r WHERE r.key = ${alias}.key AND r.layer = ${REFRESHED} AND r.path = ${alias}.path)))`;
+    return `${alias}.present = 1 AND (${alias}.layer = ${REFRESHED} OR (${alias}.layer = ${CONFIRMED} AND ${unrefreshed(alias)}))`;
+}
+function layers(view) {
+    if (view !== 'refreshed')
+        return [viewed(view, 'e')];
+    return [
+        `e.layer = ${REFRESHED} AND e.present = 1`,
+        `e.layer = ${CONFIRMED} AND e.present = 1 AND ${unrefreshed('e')}`,
+    ];
+}
+function pathOrdered(connection, columns, key, view, condition, after, limit) {
+    const beyond = after === null ? '' : ' AND e.path > ?';
+    const parts = layers(view).map((layer) => `SELECT ${columns} FROM replica_entries e WHERE e.key = ? AND ${layer}${condition}${beyond}`);
+    return connection.rows(`${parts.join(' UNION ALL ')} ORDER BY path LIMIT ?`, [
+        ...parts.flatMap(() => (after === null ? [key] : [key, after])),
+        limit,
+    ]);
 }
 export function entries(connection, key, view, list) {
     if (list.length === 0)
@@ -489,61 +461,21 @@ export function typed(connection, key, view, list) {
         binary: number(row.binary) === 1,
     }));
 }
-export function referrers(connection, key, view, targets, kinds) {
+export function referrers(connection, key, view, targets) {
     if (targets.length === 0)
         return [];
     return connection
-        .rows(`SELECT DISTINCT e.path, l.target, l.kind FROM replica_links l JOIN replica_entries e ON e.id = l.doc WHERE l.key = ? AND l.target IN (SELECT value FROM json_each(?)) AND l.kind IN (SELECT value FROM json_each(?)) AND ${viewed(view, 'e')}`, [key, paths(targets), JSON.stringify(kinds)])
-        .map((row) => ({
-        path: text(row.path),
-        target: text(row.target),
-        kind: text(row.kind),
-    }));
-}
-export function walk(connection, key, view, after, bytes) {
-    const found = [];
-    let spent = 0;
-    let from = after;
-    for (;;) {
-        const rows = connection.rows(`SELECT ${ENTRY_COLUMNS} FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')}${from === null ? '' : ' AND e.path > ?'} ORDER BY e.path LIMIT 64`, from === null ? [key] : [key, from]);
-        for (const row of rows) {
-            const entry = heldEntry(row);
-            found.push(entry);
-            spent += 256 + (entry.content?.length ?? 0) * 2;
-            if (spent >= bytes)
-                return { entries: found, next: entry.path };
-        }
-        if (rows.length < 64)
-            return { entries: found, next: null };
-        from = found.at(-1).path;
-    }
+        .rows(`SELECT DISTINCT e.path, l.target FROM replica_links l JOIN replica_entries e ON e.id = l.doc WHERE l.key = ? AND l.target IN (SELECT value FROM json_each(?)) AND ${viewed(view, 'e')}`, [key, paths(targets)])
+        .map((row) => ({ path: text(row.path), target: text(row.target) }));
 }
 export function files(connection, key, view, after, limit) {
-    return connection
-        .rows(`SELECT e.path, e.revision FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.content IS NOT NULL AND e.revision IS NOT NULL${after === null ? '' : ' AND e.path > ?'} ORDER BY e.path LIMIT ?`, after === null ? [key, limit] : [key, after, limit])
-        .map((row) => ({ path: text(row.path), revision: text(row.revision) }));
+    return pathOrdered(connection, 'e.path, e.revision', key, view, ' AND e.content IS NOT NULL AND e.revision IS NOT NULL', after, limit).map((row) => ({ path: text(row.path), revision: text(row.revision) }));
+}
+export function folders(connection, key, view, list) {
+    return list.filter((path) => connection.rows(`SELECT 1 FROM replica_folders f WHERE f.key = ? AND f.layer = ${CONFIRMED} AND f.present = 1 AND f.path = ? UNION ALL SELECT 1 FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.path >= ? AND e.path < ? LIMIT 1`, [key, path, key, `${path}/`, `${path}0`]).length > 0);
 }
 export function typedWithin(connection, key, view, roots, within) {
-    const inside = roots.map(() => '(e.path = ? OR (e.path >= ? AND e.path < ?))').join(' OR ');
-    const bind = [key];
-    for (const root of roots)
-        bind.push(root, `${root}/`, `${root}0`);
-    const placed = roots.length === 0 ? (within ? ' AND 0' : '') : ` AND ${within ? '' : 'NOT '}(${inside})`;
-    return (connection.rows(`SELECT 1 FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.declared IS NOT NULL AND e.declared <> 'any'${placed} LIMIT 1`, bind).length > 0);
-}
-export function transferPart(connection, locator, token) {
-    const [held] = connection.rows('SELECT part FROM replica_transfer_parts WHERE locator = ? AND token = ?', [locator, token]);
-    return held?.part instanceof Uint8Array ? held.part : null;
-}
-export function keepParts(connection, parts) {
-    for (const { locator, token, part } of parts) {
-        connection.run('DELETE FROM replica_transfer_parts WHERE locator = ? AND token = ?', [
-            locator,
-            token,
-        ]);
-        connection.run('INSERT INTO replica_transfer_parts (locator, token, part) SELECT ?, ?, ? WHERE EXISTS (SELECT 1 FROM replica_transfers WHERE locator = ?)', [locator, token, part, locator]);
-    }
-    return null;
+    return (connection.rows(`SELECT 1 FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.declared IS NOT NULL AND e.declared <> 'any' AND ${within ? '' : 'NOT '}${beneathRoots('e.path')} LIMIT 1`, [key, JSON.stringify(roots)]).length > 0);
 }
 export function opened(connection) {
     const facts = {};
@@ -551,31 +483,18 @@ export function opened(connection) {
         facts[text(row.name)] = text(row.value);
     }
     return {
-        rooms: connection.rows('SELECT locator, key, record FROM replica_rooms').map((row) => ({
+        rooms: connection.rows(HELD_RECORDS).map((row) => ({
             locator: text(row.locator),
             key: number(row.key),
             record: text(row.record),
         })),
-        transfers: connection.rows('SELECT locator, key, record FROM replica_transfers').map((row) => ({
-            locator: text(row.locator),
-            key: number(row.key),
-            record: text(row.record),
-        })),
-        kept: connection
-            .rows('SELECT locator, token, length(part) AS bytes FROM replica_transfer_parts')
-            .map((row) => ({
-            locator: text(row.locator),
-            token: text(row.token),
-            bytes: number(row.bytes),
-        })),
         keys: connection
-            .rows('SELECT key, locator, transfer, rederiving, searching FROM replica_keys')
+            .rows('SELECT key, locator, transfer, rederiving FROM replica_keys')
             .map((row) => ({
             key: number(row.key),
             locator: text(row.locator),
             transfer: text(row.transfer),
             rederiving: cursor(row.rederiving),
-            searching: cursor(row.searching),
         })),
         facts,
         presentation: connection.rows('SELECT app, value FROM replica_presentation').map((row) => ({
@@ -588,7 +507,6 @@ export function opened(connection) {
         orphans: connection
             .rows(`SELECT k.key FROM replica_keys k
          WHERE NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
-           AND NOT EXISTS (SELECT 1 FROM replica_transfers t WHERE t.key = k.key)
            AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)`)
             .map((row) => number(row.key)),
         erasing: connection.rows('SELECT key, locator FROM replica_erasing').map((row) => ({
@@ -603,5 +521,4 @@ export function ended(connection) {
         if (table !== undefined)
             connection.run(`DELETE FROM ${table}`);
     }
-    unindexAll(connection);
 }