@agentshouse/mdruntime 0.1.0-cee9c8aa4cd86 → 0.1.0-cf1e7fdb9c600

package/dist/compile.js

@@ -1,107 +1,36 @@
-import { ARRAY_INDEX, parse, pointerSegments } from '@agentshouse/mdmodel';
+import { ARRAY_INDEX, bodied, byCodePoint, compared, cursorOf, heldKind, parse, pointerSegments, sql, } from '@agentshouse/mdmodel';
 import { binaryRecord } from './binary.js';
-import { documentKey, kindOf, pathOfKey, roomOfKey, settingsTextPath } from './rows.js';
-import { byCodePoint, compared, cursorOf, FROM_START, MAXIMUM_WINDOW, } from './windows.js';
+import { documentKey, pathOfKey, roomOfKey } from './rows.js';
+import { FROM_START, MAXIMUM_WINDOW } from './windows.js';
+const { all, any, exists, fragment, grouped, joined, listed } = sql;
 const VISIBLE = (alias) => `${alias}.visible = 1 AND ${alias}.present = 1 AND ${alias}.queried = 1`;
-function fragment(sql, bind = []) {
-    return { sql, bind };
+function located(alias) {
+    return fragment(`(SELECT k.locator FROM replica_keys k WHERE k.key = ${alias}.key)`);
 }
-function joined(parts, separator) {
-    return {
-        sql: parts.map((part) => part.sql).join(separator),
-        bind: parts.flatMap((part) => part.bind),
-    };
-}
-function all(parts) {
-    return parts.length === 0 ? fragment('1') : joined(parts.map(grouped), ' AND ');
-}
-function any(parts) {
-    return parts.length === 0 ? fragment('0') : joined(parts.map(grouped), ' OR ');
-}
-function grouped(part) {
-    return { sql: `(${part.sql})`, bind: part.bind };
-}
-function list(values) {
-    return fragment(`(${values.map(() => '?').join(', ')})`, values);
-}
-function sqlValue(value) {
-    if (value === null)
-        return { kind: 'null', value: null };
-    if (typeof value === 'boolean')
-        return { kind: 'boolean', value: value ? 1 : 0 };
-    if (typeof value === 'number')
-        return { kind: 'number', value };
-    return { kind: 'string', value };
-}
-function same(alias, value) {
-    const held = sqlValue(value);
-    return held.kind === 'null'
-        ? fragment(`${alias}.kind = 'null'`)
-        : fragment(`${alias}.kind = ? AND ${alias}.value = ?`, [held.kind, held.value]);
-}
-const BOUNDS = { gt: '>', gte: '>=', lt: '<', lte: '<=' };
-function heldValues(doc, field, alias, where) {
-    return fragment(`SELECT 1 FROM replica_values ${alias} WHERE ${alias}.doc = ${doc}.id AND ${alias}.form = ? AND (${where.sql})`, [field, ...where.bind]);
-}
-function exists(part) {
-    return { sql: `EXISTS (${part.sql})`, bind: part.bind };
-}
-function matching(held, doc, field, values) {
-    return any(values.map((value) => any([
-        same(held, value),
-        all([
-            fragment(`${held}.kind = 'array'`),
-            exists(heldValues(doc, field, 'i', all([fragment(`i.held = ${held}.held AND i.item >= 0`), same('i', value)]))),
-        ]),
-    ])));
-}
-function tested(doc, field, test) {
-    const reached = exists(heldValues(doc, field, 'r', fragment('r.item = -1')));
-    if (test.op === 'exists')
-        return test.exists ? reached : fragment(`NOT ${reached.sql}`, reached.bind);
-    if (test.op === 'eq' || test.op === 'in') {
-        const values = test.op === 'eq' ? [test.value] : test.values;
-        return exists(heldValues(doc, field, 'v', any(values.map((value) => same('v', value)))));
-    }
-    if (test.op === 'ne' || test.op === 'nin') {
-        const values = test.op === 'ne' ? [test.value] : test.values;
-        const unmatched = matching('h', doc, field, values);
-        return any([
-            fragment(`NOT ${reached.sql}`, reached.bind),
-            exists(heldValues(doc, field, 'h', all([fragment('h.item = -1'), fragment(`NOT (${unmatched.sql})`, unmatched.bind)]))),
-        ]);
-    }
-    if (test.op === 'range') {
-        return exists(heldValues(doc, field, 'v', all([
-            fragment('v.item = -1'),
-            ...test.bounds.map(([operator, bound]) => {
-                const held = sqlValue(bound);
-                return fragment(`v.kind = ? AND v.value ${BOUNDS[operator]} ?`, [
-                    held.kind,
-                    held.value,
-                ]);
-            }),
-        ])));
-    }
-    if (test.op === 'all') {
-        return exists(heldValues(doc, field, 'h', all([
-            fragment("h.item = -1 AND h.kind = 'array'"),
-            ...test.values.map((value) => exists(heldValues(doc, field, 'i', all([fragment('i.held = h.held AND i.item >= 0'), same('i', value)])))),
-        ])));
-    }
-    return exists(heldValues(doc, field, 'v', fragment("v.item = -1 AND v.kind = 'array' AND v.size = ?", [test.size])));
+function within(alias, path) {
+    return fragment(`(${alias}.path = ? OR (${alias}.path >= ? AND ${alias}.path < ?))`, [
+        path,
+        `${path}/`,
+        `${path}0`,
+    ]);
 }
-function conditioned(doc, condition) {
-    if ('every' in condition)
-        return all(condition.every.map((one) => conditioned(doc, one)));
-    if ('some' in condition)
-        return any(condition.some.map((one) => conditioned(doc, one)));
-    return tested(doc, condition.field, condition.test);
+function ancestors(area) {
+    const depth = area.path.split('/').length;
+    return area.open.flatMap((open) => {
+        const segments = open.split('/');
+        return segments.slice(depth - 1, -1).map((_, at) => segments.slice(0, depth + at).join('/'));
+    });
 }
-function located(bound, alias) {
-    if (bound.rooms.length === 0)
-        return fragment("''");
-    return fragment(`CASE ${alias}.key ${bound.rooms.map(() => 'WHEN ? THEN ?').join(' ')} END`, bound.rooms.flatMap(({ key, locator }) => [key, locator]));
+function fenced(alias, key, area, folders) {
+    const opened = any([
+        ...area.open.map((open) => within(alias, open)),
+        ...(folders ? ancestors(area).map((path) => fragment(`${alias}.path = ?`, [path])) : []),
+    ]);
+    return all([
+        fragment(`${alias}.key = ?`, [key]),
+        within(alias, area.path),
+        fragment(`NOT ${grouped(opened).sql}`, opened.bind),
+    ]);
 }
 function reached(bound, alias) {
     const reach = bound.reach;
@@ -115,44 +44,38 @@ function reached(bound, alias) {
         const key = bound.keys.get(roomOfKey(held));
         return key === undefined ? [] : [[key, pathOfKey(held)]];
     });
-    return any([
-        ...(rooms.length === 0 ? [] : [fragment(`${alias}.key IN ${list(rooms).sql}`, rooms)]),
-        ...(documents.length === 0
-            ? []
-            : [
-                fragment(`EXISTS (SELECT 1 FROM json_each(?) j WHERE j.value ->> 0 = ${alias}.key AND j.value ->> 1 = ${alias}.path)`, [JSON.stringify(documents)]),
-            ]),
-    ]);
-}
-function scoped(alias, scope) {
+    const outside = any(reach.areas.flatMap((area) => {
+        const key = bound.keys.get(area.room);
+        return key === undefined ? [] : [fenced(alias, key, area, false)];
+    }));
     return all([
-        ...(scope.types === null
-            ? []
-            : [fragment(`${alias}.type IN ${list(scope.types).sql}`, scope.types)]),
-        ...(scope.pathPrefix === null || scope.pathPrefix === ''
-            ? []
-            : [
-                fragment(`${alias}.path >= ? AND ${alias}.path < ?`, [
-                    `${scope.pathPrefix}/`,
-                    `${scope.pathPrefix}0`,
+        any([
+            ...(rooms.length === 0 ? [] : [fragment(`${alias}.key IN ${listed(rooms).sql}`, rooms)]),
+            ...(documents.length === 0
+                ? []
+                : [
+                    fragment(`EXISTS (SELECT 1 FROM json_each(?) j WHERE j.value ->> 0 = ${alias}.key AND j.value ->> 1 = ${alias}.path)`, [JSON.stringify(documents)]),
                 ]),
-            ]),
-        ...(scope.where === null ? [] : [conditioned(alias, scope.where)]),
+        ]),
+        ...(reach.areas.length === 0 ? [] : [fragment(`NOT ${grouped(outside).sql}`, outside.bind)]),
     ]);
 }
 function related(bound, plan) {
     const parts = [];
     if (plan.hop !== null) {
         parts.push(exists(joined([
-            fragment(`SELECT 1 FROM replica_links l JOIN replica_entries m ON m.key = d.key AND m.path = l.target AND ${VISIBLE('m')} WHERE l.doc = d.id AND l.kind = 'field' AND l.form = ? AND`, [plan.hop.field]),
-            grouped(all([scoped('m', plan.hop), reached(bound, 'm')])),
+            fragment(`SELECT 1 FROM replica_links l JOIN replica_entries m ON m.key = d.key AND m.path = l.target AND ${VISIBLE('m')} WHERE l.doc = d.id AND l.kind = 'field' AND`),
+            grouped(sql.placed('l', plan.hop.field, 'link')),
+            fragment('AND'),
+            grouped(all([sql.scoped('m', plan.hop), reached(bound, 'm')])),
         ], ' ')));
     }
     if (plan.backlinks !== null) {
         const field = plan.backlinks.field;
         parts.push(exists(joined([
-            fragment(`SELECT 1 FROM replica_links l JOIN replica_entries s ON s.id = l.doc AND ${VISIBLE('s')} WHERE l.key = d.key AND l.target = d.path AND l.kind = 'field'${field === null ? '' : ' AND l.form = ?'} AND`, field === null ? [] : [field]),
-            grouped(all([scoped('s', plan.backlinks), reached(bound, 's')])),
+            fragment(`SELECT 1 FROM replica_links l JOIN replica_entries s ON s.id = l.doc AND ${VISIBLE('s')} WHERE l.key = d.key AND l.target = d.path AND l.kind = 'field' AND`),
+            ...(field === null ? [] : [grouped(sql.placed('l', field, 'link')), fragment('AND')]),
+            grouped(all([sql.scoped('s', plan.backlinks), reached(bound, 's')])),
         ], ' ')));
     }
     return parts;
@@ -160,11 +83,9 @@ function related(bound, plan) {
 function filtered(bound, plan) {
     const keys = bound.rooms.map(({ key }) => key);
     return all([
-        fragment(`d.key IN ${list(keys).sql} AND ${VISIBLE('d')}`, keys),
-        ...(plan.paths === null ? [] : [fragment(`d.path IN ${list(plan.paths).sql}`, plan.paths)]),
-        ...(plan.folder === null ? [] : [fragment('d.folder = ?', [plan.folder])]),
+        fragment(`d.key IN ${listed(keys).sql} AND ${VISIBLE('d')}`, keys),
         reached(bound, 'd'),
-        scoped('d', plan.root),
+        sql.tests('d', plan),
         ...related(bound, plan),
     ]);
 }
@@ -195,99 +116,25 @@ const GROUP_TIES = {
     locale: undefined,
     localeOptions: undefined,
 };
-function classOf(value, ordering) {
-    if (value === null || value === undefined) {
-        return { placement: ordering.nulls === 'first' ? 0 : 2, klass: 0, scalar: null };
-    }
-    if (typeof value === 'boolean')
-        return { placement: 1, klass: 1, scalar: value ? 1 : 0 };
-    if (typeof value === 'number')
-        return { placement: 1, klass: 1, scalar: value };
-    if (typeof value === 'string')
-        return { placement: 1, klass: 2, scalar: value };
-    return { placement: 1, klass: 3, scalar: JSON.stringify(value) };
-}
-function placement(term) {
-    const nulls = term.ordering.nulls === 'first' ? 0 : 2;
-    return fragment(`CASE WHEN ${term.kind.sql} IS NULL OR ${term.kind.sql} = 'null' THEN ${nulls} ELSE 1 END`, [...term.kind.bind, ...term.kind.bind]);
-}
-function klass(term) {
-    return fragment(`CASE ${term.kind.sql} WHEN 'boolean' THEN 1 WHEN 'number' THEN 1 WHEN 'string' THEN 2 WHEN 'array' THEN 3 ELSE 4 END`, term.kind.bind);
-}
-function classed(term, which) {
-    const kinds = which === 1 ? "('boolean', 'number')" : which === 2 ? "('string')" : "('array')";
-    return fragment(`(CASE WHEN ${term.kind.sql} IN ${kinds} THEN ${term.value.sql} END)`, [
-        ...term.kind.bind,
-        ...term.value.bind,
-    ]);
-}
-function collated(bound, term, which) {
-    if (which === 1)
-        return '';
-    const name = bound.collate(term.ordering, which === 3, term.total === true);
-    return name === null ? '' : ` COLLATE ${name}`;
-}
-function sorted(bound, term) {
-    const direction = term.ordering.direction === 'desc' ? 'DESC' : 'ASC';
-    return joined([
-        placement(term),
-        fragment(`${klass(term).sql} ${direction}`, klass(term).bind),
-        ...[1, 2, 3].map((which) => {
-            const value = classed(term, which);
-            return fragment(`${value.sql}${collated(bound, term, which)} ${direction}`, value.bind);
-        }),
-    ], ', ');
-}
-function beyond(bound, term, value) {
-    const held = classOf(value, term.ordering);
-    const place = placement(term);
-    if (held.placement !== 1) {
-        return {
-            beyond: fragment(`${place.sql} > ${held.placement}`, place.bind),
-            equal: fragment(`${place.sql} = ${held.placement}`, place.bind),
-        };
-    }
-    const after = term.ordering.direction === 'desc' ? '<' : '>';
-    const kind = klass(term);
-    const scalar = classed(term, held.klass);
-    const collation = collated(bound, term, held.klass);
-    return {
-        beyond: any([
-            fragment(`${place.sql} > 1`, place.bind),
-            all([
-                fragment(`${place.sql} = 1`, place.bind),
-                any([
-                    fragment(`${kind.sql} ${after} ${held.klass}`, kind.bind),
-                    all([
-                        fragment(`${kind.sql} = ${held.klass}`, kind.bind),
-                        fragment(`${scalar.sql} ${after} ?${collation}`, [...scalar.bind, held.scalar]),
-                    ]),
-                ]),
-            ]),
-        ]),
-        equal: all([
-            fragment(`${place.sql} = 1 AND ${kind.sql} = ${held.klass}`, [...place.bind, ...kind.bind]),
-            fragment(`${scalar.sql} = ?${collation}`, [...scalar.bind, held.scalar]),
-        ]),
-    };
-}
-function following(parts, tie) {
-    let condition = tie;
-    for (const part of [...parts].reverse()) {
-        condition = any([part.beyond, all([part.equal, condition])]);
-    }
-    return condition;
-}
-function keyTerm(bound, key, ordering, index) {
+function keyTerm(key, ordering, index) {
     if (key.of === 'field') {
         const alias = `o${index}`;
         return {
-            term: { ordering, kind: fragment(`${alias}.kind`), value: fragment(`${alias}.value`) },
-            join: fragment(`LEFT JOIN replica_values ${alias} ON ${alias}.doc = d.id AND ${alias}.form = ? AND ${alias}.single = 1 AND ${alias}.item = -1`, [key.field]),
+            term: {
+                ordering,
+                kind: fragment(`${alias}.kind`),
+                value: fragment(`${alias}.value`),
+                instant: fragment(`${alias}.instant`),
+            },
+            join: joined([
+                fragment(`LEFT JOIN replica_values ${alias} ON ${alias}.doc = d.id AND `),
+                grouped(sql.placed(alias, key.field, 'exact')),
+                fragment(` AND ${alias}.item = -1 AND ${alias}.kind NOT IN ('object', 'undefined')`),
+            ], ''),
         };
     }
     const expression = key.of === 'room'
-        ? located(bound, 'd')
+        ? located('d')
         : key.of === 'path'
             ? fragment('d.path')
             : key.of === 'type'
@@ -313,13 +160,20 @@ function decoded(kind, value) {
         return JSON.parse(value);
     return null;
 }
+function sortedValue(row, index) {
+    const kind = row[`k${index}`];
+    return sql.sortValue(kind, decoded(kind, row[`v${index}`]), row[`i${index}`]);
+}
 function decodedKey(key, row, index) {
     if (key.of === 'field') {
         const kind = row[`k${index}`];
-        return kind === null || kind === undefined ? undefined : decoded(kind, row[`v${index}`]);
+        return kind === null || kind === undefined ? undefined : sortedValue(row, index);
     }
     return row[`v${index}`] ?? null;
 }
+function instantOf(term) {
+    return term.instant ?? fragment('NULL');
+}
 const registered = new WeakMap();
 function hashed(text) {
     let hash = 0x811c9dc5;
@@ -369,9 +223,6 @@ function collating(connection) {
         return name;
     };
 }
-function bodied(shape) {
-    return shape.sections !== null || shape.preamble || shape.markdown;
-}
 function masked(pointers) {
     const root = { whole: false, members: new Map() };
     for (const pointer of pointers) {
@@ -427,24 +278,26 @@ function documentOf(shape, room, row, linked) {
     const path = row.path;
     const type = row.type;
     const fileId = row.file_id ?? null;
+    const revision = row.revision ?? null;
+    const content = row.content ?? null;
+    const unheld = Number(row.unheld) === 1 ? { textHeld: false } : {};
     if (type === null) {
         const binary = row.binary === null ? null : JSON.parse(row.binary);
-        const content = row.content ?? null;
         return {
             room,
             path,
             ...(binary === null
                 ? {}
                 : {
-                    binaryFile: binaryRecord(path, binary, row.revision ?? undefined),
+                    binaryFile: binaryRecord(path, binary, revision ?? undefined),
                 }),
-            ...(shape.markdown && content !== null && settingsTextPath(path)
-                ? { markdown: content }
-                : {}),
+            ...(shape.markdown && content !== null ? { markdown: content } : {}),
+            ...unheld,
             ...(shape.fileId ? { fileId } : {}),
+            ...(shape.revision ? { revision } : {}),
         };
     }
-    const read = bodied(shape) ? parse(row.content, path) : null;
+    const read = bodied(shape) && content !== null ? parse(content, path) : null;
     const body = read?.ok === true ? read.document : null;
     const fields = JSON.parse(row.fields ?? '{}');
     return {
@@ -462,7 +315,8 @@ function documentOf(shape, room, row, linked) {
             ? {}
             : { sections: sectionsOf(body.sections, shape.sections) }),
         ...(shape.preamble && body !== null ? { preamble: body.preamble } : {}),
-        ...(shape.markdown && body !== null ? { markdown: body.content } : {}),
+        ...(shape.markdown && content !== null ? { markdown: content } : {}),
+        ...unheld,
         ...(shape.backlinks ? { backlinks: linked.get(documentKey(room, path)) ?? [] } : {}),
         ...(shape.change
             ? {
@@ -470,6 +324,7 @@ function documentOf(shape, room, row, linked) {
             }
             : {}),
         ...(shape.fileId ? { fileId } : {}),
+        ...(shape.revision ? { revision } : {}),
     };
 }
 function backlinksOf(connection, bound, rows) {
@@ -495,10 +350,10 @@ function backlinksOf(connection, bound, rows) {
     return linked;
 }
 function documents(connection, bound, plan, shape, budget) {
-    const terms = shape.orderBy.map(({ key, options }, index) => keyTerm(bound, key, orderingOf(options), index));
+    const terms = shape.orderBy.map(({ key, options }, index) => keyTerm(key, orderingOf(options), index));
     const filter = filtered(bound, plan);
     const cursor = plan.after?.values ?? null;
-    const room = located(bound, 'd');
+    const room = located('d');
     const tie = cursor === null
         ? null
         : (() => {
@@ -514,19 +369,20 @@ function documents(connection, bound, plan, shape, budget) {
         })();
     const after = cursor === null || tie === null
         ? null
-        : following(terms.map(({ term }, index) => beyond(bound, term, cursor[index])), tie);
+        : sql.following(terms.map(({ term }, index) => sql.beyond(bound.collate, term, cursor[index])), tie);
     const selected = joined([
-        fragment('SELECT d.id, d.key, d.path, d.type, d.binary, d.revision, d.fields, d.change, d.file_id'),
+        fragment('SELECT d.id, d.key, d.path, d.type, d.binary, d.revision, d.fields, d.change, d.file_id, d.content IS NULL AND d.outline IS NOT NULL AS unheld'),
         ...terms.map(({ term }, index) => joined([
             fragment(`, ${term.kind.sql} AS k${index}`, term.kind.bind),
             fragment(`, ${term.value.sql} AS v${index}`, term.value.bind),
+            fragment(`, ${instantOf(term).sql} AS i${index}`, instantOf(term).bind),
         ], '')),
         fragment(' FROM replica_entries d'),
         ...terms.flatMap(({ join }) => (join === null ? [] : [fragment(' '), join])),
         fragment(' WHERE '),
         after === null ? filter : all([filter, after]),
         fragment(' ORDER BY '),
-        joined([...terms.map(({ term }) => sorted(bound, term)), room, fragment('d.path')], ', '),
+        joined([...terms.map(({ term }) => sql.sorted(bound.collate, term)), room, fragment('d.path')], ', '),
         ...(plan.limit === null ? [] : [fragment(' LIMIT ?', [plan.limit + 1])]),
     ], '');
     const rows = connection.rows(selected.sql, selected.bind);
@@ -565,7 +421,7 @@ function withContent(connection, rows, budget) {
         return rows;
     const ids = rows.map((row) => row.id);
     const sizes = new Map(connection
-        .rows(`SELECT id, octet_length(content) AS size FROM replica_entries WHERE id IN ${list(ids).sql}`, ids)
+        .rows(`SELECT id, octet_length(content) AS size FROM replica_entries WHERE id IN ${listed(ids).sql}`, ids)
         .map((row) => [row.id, Number(row.size ?? 0)]));
     let cut = rows.length;
     let spent = 0;
@@ -580,20 +436,30 @@ function withContent(connection, rows, budget) {
     const kept = rows.slice(0, cut);
     const read = kept.map((row) => row.id);
     const held = new Map(connection
-        .rows(`SELECT id, content FROM replica_entries WHERE id IN ${list(read).sql}`, read)
+        .rows(`SELECT id, content FROM replica_entries WHERE id IN ${listed(read).sql}`, read)
         .map((row) => [row.id, row.content]));
     return kept.map((row) => ({ ...row, content: held.get(row.id) ?? null }));
 }
 function storedFolders(bound, plan) {
-    if (plan.paths !== null || plan.hop !== null || plan.backlinks !== null)
+    if (plan.paths !== null || plan.fileIds !== null)
+        return null;
+    if (plan.hop !== null || plan.backlinks !== null)
         return null;
     const whole = bound.rooms.filter(({ locator }) => bound.reach === null || bound.reach.rooms.includes(locator));
     if (whole.length === 0)
         return null;
     const keys = whole.map(({ key }) => key);
     const prefix = plan.root.pathPrefix;
+    const outside = (bound.reach?.areas ?? []).flatMap((area) => {
+        const key = bound.keys.get(area.room);
+        if (key === undefined)
+            return [];
+        const fence = fenced('f', key, area, true);
+        return [fragment(`NOT ${grouped(fence).sql}`, fence.bind)];
+    });
     return all([
-        fragment(`f.key IN ${list(keys).sql} AND f.layer = 0 AND f.present = 1`, keys),
+        fragment(`f.key IN ${listed(keys).sql} AND f.layer = 0 AND f.present = 1`, keys),
+        ...outside,
         ...(prefix === null || prefix === ''
             ? []
             : [
@@ -608,7 +474,7 @@ function storedFolders(bound, plan) {
 }
 function groups(connection, bound, plan, shape) {
     const keyed = shape.groupBy.map((key, index) => {
-        const { term, join } = keyTerm(bound, key, GROUP_ORDER, index);
+        const { term, join } = keyTerm(key, GROUP_ORDER, index);
         const kind = fragment(`COALESCE(${term.kind.sql}, 'null')`, term.kind.bind);
         return { term: { ...term, kind }, join };
     });
@@ -618,6 +484,7 @@ function groups(connection, bound, plan, shape) {
         joined(keyed.map(({ term }, index) => joined([
             fragment(`${term.kind.sql} AS k${index}`, term.kind.bind),
             fragment(`, ${term.value.sql} AS v${index}`, term.value.bind),
+            fragment(`, ${instantOf(term).sql} AS i${index}`, instantOf(term).bind),
         ], '')), ', '),
         fragment(', 1 AS n FROM replica_entries d'),
         ...keyed.flatMap(({ join }) => (join === null ? [] : [fragment(' '), join])),
@@ -631,10 +498,15 @@ function groups(connection, bound, plan, shape) {
         ? inner
         : joined([
             inner,
-            fragment(" UNION ALL SELECT 'string' AS k0, f.path AS v0, 0 AS n FROM replica_folders f WHERE "),
+            fragment(" UNION ALL SELECT 'string' AS k0, f.path AS v0, NULL AS i0, 0 AS n FROM replica_folders f WHERE "),
             folders,
         ], '');
-    const columns = keyed.map((_, index) => `k${index}, v${index}`).join(', ');
+    const columns = keyed
+        .map((_, index) => `k${index}, min(v${index}) AS v${index}, i${index}`)
+        .join(', ');
+    const held = keyed
+        .map((_, index) => `k${index}, i${index}, CASE WHEN i${index} IS NULL THEN v${index} END`)
+        .join(', ');
     const counted = {
         ordering: { ...GROUP_ORDER },
         kind: fragment("'number'"),
@@ -647,7 +519,7 @@ function groups(connection, bound, plan, shape) {
     });
     const ordered = shape.orderBy.map(({ by, direction }) => by === 'count'
         ? { ...counted, ordering: { ...GROUP_ORDER, direction } }
-        : outer(by, { ...GROUP_ORDER, direction }));
+        : { ...outer(by, { ...GROUP_ORDER, direction }), instant: fragment(`i${by}`) });
     const ties = keyed.flatMap((_, index) => [
         { ...outer(index, GROUP_TIES), total: true },
         { ordering: GROUP_TIES, kind: fragment("'string'"), value: fragment(`k${index}`) },
@@ -657,19 +529,21 @@ function groups(connection, bound, plan, shape) {
         ? null
         : cursor[shape.orderBy.length].flatMap((value) => [
             value,
-            kindOf(value ?? null),
+            heldKind(value ?? null),
         ]);
     const after = cursor === null || tieValues === null
         ? null
-        : following(ordered.map((term, index) => beyond(bound, term, cursor[index])), following(ties.slice(0, -1).map((term, index) => beyond(bound, term, tieValues[index])), beyond(bound, ties.at(-1), tieValues.at(-1)).beyond));
-    const grouping = joined([fragment(`SELECT ${columns}, sum(n) AS n FROM (`), source, fragment(`) GROUP BY ${columns}`)], '');
+        : sql.following(ordered.map((term, index) => sql.beyond(bound.collate, term, cursor[index])), sql.following(ties
+            .slice(0, -1)
+            .map((term, index) => sql.beyond(bound.collate, term, tieValues[index])), sql.beyond(bound.collate, ties.at(-1), tieValues.at(-1)).beyond));
+    const grouping = joined([fragment(`SELECT ${columns}, sum(n) AS n FROM (`), source, fragment(`) GROUP BY ${held}`)], '');
     const windowed = joined([
         fragment('SELECT * FROM ('),
         grouping,
         fragment(')'),
         ...(after === null ? [] : [fragment(' WHERE '), after]),
         fragment(' ORDER BY '),
-        joined([...ordered, ...ties].map((term) => sorted(bound, term)), ', '),
+        joined([...ordered, ...ties].map((term) => sql.sorted(bound.collate, term)), ', '),
         ...(plan.limit === null ? [] : [fragment(' LIMIT ?', [plan.limit + 1])]),
     ], '');
     const rows = connection.rows(windowed.sql, windowed.bind);
@@ -682,7 +556,7 @@ function groups(connection, bound, plan, shape) {
         : last === undefined
             ? (plan.after?.text ?? FROM_START)
             : cursorOf([
-                ...shape.orderBy.map(({ by }) => by === 'count' ? Number(last.n) : decoded(last[`k${by}`], last[`v${by}`])),
+                ...shape.orderBy.map(({ by }) => by === 'count' ? Number(last.n) : sortedValue(last, by)),
                 keyOf(last),
             ]);
     const total = plan.total
@@ -702,9 +576,6 @@ function boundOf(connection, plan, rooms) {
         collate: collating(connection),
     };
 }
-export function selected(connection, plan, rooms) {
-    return filtered(boundOf(connection, plan, rooms), plan);
-}
 export function answerQuery(connection, plan, rooms, budget = null) {
     const bound = boundOf(connection, plan, rooms);
     if (rooms.length === 0) {

package/dist/connection.d.ts

@@ -1,20 +1,20 @@
+import { type DocumentQuery, type GroupQuery } from '@agentshouse/mdmodel';
 import { type TypesResult } from './model.js';
 import { type Observation } from './observation.js';
-import { type DocumentQuery, type DocumentResult, type GroupQuery, type GroupResult, type PendingResult } from './query.js';
+import { type DocumentResult, type GroupResult, type PendingResult } from './query.js';
 import { type EntrySeed } from './seed.js';
 import { type SearchRequest, type SearchResult } from './search.js';
 import { type Adapter, type Authorization, type Member, type ProviderRefusal, type ServiceMode } from './adapter.js';
 import { type ChatDraftSurface, type ConversationAgent, type ConversationEntry, type ConversationOptions, type ConversationSurface, type ConversationView } from './conversation.js';
 import { type WrittenBy } from './change.js';
 import type { Open } from './open.js';
-import { type BinaryReference } from './binary.js';
-import { type BinaryLinkAnswer, type CallResult, type HeldBinaryAnswer, type HistoryAnswer, type QueueEntry, type Resolution } from './outcome.js';
+import { type AttachAnswer, type BinaryLinkAnswer, type CallResult, type HeldBinaryAnswer, type HeldShownAnswer, type HistoryAnswer, type QueueEntry, type Resolution } from './outcome.js';
 import { type Recovery } from './recovery.js';
 import { type SaveRefusal } from './save.js';
 import { type RoomDraft, type RoomDraftParts, type RoomDraftWrite } from './room-draft.js';
 import { type SyncStatus } from './status.js';
 import { type RuntimeShell } from './durability.js';
-import { type CapacityCause, type CodeAvailability, type Persistence, type Readiness, type SearchObligation } from './readiness.js';
+import { type CodeAvailability, type Persistence, type Readiness } from './readiness.js';
 import type { RuntimeDatabase } from './tables.js';
 import { type NarrowedView, type PortReach, type RuntimeView } from './reach.js';
 export type AppTypedSelection = {
@@ -55,6 +55,7 @@ export type MdRuntimeOptions = {
     authorization?: Authorization;
     signal?: AbortSignal;
     recovery?: Recovery;
+    usableWhenHeld?: readonly string[];
 };
 export type SaveResult = {
     outcome: 'saved';
@@ -73,7 +74,7 @@ export type SavedParts = {
 export type RoomDraftSurface = {
     list(): Promise<readonly RoomDraft[]>;
     read(room: string, path: string): Promise<RoomDraft | null>;
-    write(room: string, path: string, revision: string | null, parts: unknown): Promise<RoomDraftWrite>;
+    write(room: string, path: string, revision: string | null, parts: unknown, next?: string): Promise<RoomDraftWrite>;
     discard(room: string, path: string, revision: string): Promise<RoomDraftWrite>;
 };
 export type PresentationSurface = {
@@ -84,7 +85,8 @@ export type PresentationSurface = {
 export type BinaryFileSurface = {
     read(room: string, path: string, revision?: string): Promise<HeldBinaryAnswer>;
     link(room: string, path: string, revision?: string): Promise<BinaryLinkAnswer>;
-    readReference(reference: BinaryReference): Promise<HeldBinaryAnswer>;
+    show(room: string, path: string, revision?: string): Promise<HeldShownAnswer>;
+    attach(room: string, file: File): Promise<AttachAnswer>;
     part(file: string, part: number): Promise<Uint8Array | null>;
 };
 export type HistoryQuery = {
@@ -93,7 +95,7 @@ export type HistoryQuery = {
 };
 export type MdRuntime = {
     loaded(): Promise<void>;
-    rooms(): readonly AppRoom[];
+    rooms(): Promise<readonly AppRoom[]>;
     query: {
         (query: DocumentQuery): Promise<DocumentResult | PendingResult>;
         (query: GroupQuery): Promise<GroupResult | PendingResult>;
@@ -123,6 +125,7 @@ export type MdRuntime = {
         (topic: 'sync-status'): Promise<Observation<SyncStatus>>;
         (topic: 'service-mode'): Promise<Observation<ServiceMode>>;
         (topic: 'member'): Promise<Observation<Member | null>>;
+        (topic: 'authority-changes'): Promise<Observation<number>>;
         (topic: 'query', query: DocumentQuery): Promise<Observation<DocumentResult | PendingResult>>;
         (topic: 'query', query: GroupQuery): Promise<Observation<GroupResult | PendingResult>>;
         (topic: 'queue', locator: string): Promise<Observation<readonly QueueEntry[]>>;
@@ -134,17 +137,14 @@ export type MdRuntime = {
     };
     reauthorize(): Promise<void>;
 };
-export type RuntimeTopic = 'rooms' | 'readiness' | 'connection-refusal' | 'storage-loss' | 'sync-status' | 'service-mode' | 'member' | 'query' | 'queue' | 'room-drafts' | 'presentation' | 'conversation' | 'conversations' | 'agents';
+export type RuntimeTopic = 'rooms' | 'readiness' | 'connection-refusal' | 'storage-loss' | 'sync-status' | 'service-mode' | 'member' | 'authority-changes' | 'query' | 'queue' | 'room-drafts' | 'presentation' | 'conversation' | 'conversations' | 'agents';
 export type ReadinessPort = {
     readonly target: string;
     readonly code: CodeAvailability;
-    readonly capacity: CapacityCause | null;
-    readonly search: SearchObligation;
 };
 export type ComposedRuntime = RuntimeView & {
     seed(entry: EntrySeed): void;
     stop(): Promise<void>;
-    lose(locators: readonly string[]): Promise<void>;
     persisted(answer: Persistence): void;
     within(reach: PortReach | null, port: ReadinessPort): NarrowedView;
 };