@agentshouse/mdruntime 0.1.0-cefbc8bb8a1c0 → 0.1.0-cf1e7fdb9c600

package/dist/collection.js

@@ -1,8 +1,7 @@
 import { parse, VERSION } from '@agentshouse/mdmodel';
 import { beneath, normalize } from './paths.js';
 import { hashed } from './observation.js';
-import { keysIn, sortedKeys } from './shared.js';
-import { ROW_WEIGHT } from './turn.js';
+import { sortedKeys } from './shared.js';
 export const MD_MODEL = { identity: '@agentshouse/mdmodel', version: VERSION };
 const typesPrinted = new WeakMap();
 function typesPrint(model) {
@@ -61,21 +60,6 @@ export function* documentsOf(collection) {
         }
     })();
 }
-export function* unreadable(text, paths = null) {
-    const named = paths ?? (yield* keysIn(text));
-    let least = null;
-    for (const path of named) {
-        const content = Object.hasOwn(text, path) ? text[path] : undefined;
-        if (content === undefined)
-            continue;
-        const read = parse(content, path);
-        if (!read.ok && (least === null || path < least.path)) {
-            least = { path, refused: { kind: 'refused', refusal: read.refusal } };
-        }
-        yield ROW_WEIGHT + content.length;
-    }
-    return least?.refused ?? null;
-}
 export function validRoot(root) {
     return typeof root === 'string' && normalize(root) === root;
 }

package/dist/compile.d.ts

@@ -1,16 +1,11 @@
 import type { Plan, QueryRows } from './query.js';
-import type { Connection, Value } from './tables.js';
+import type { Connection } from './tables.js';
 export type QueryRoom = {
     readonly locator: string;
     readonly key: number;
 };
-export type Fragment = {
-    readonly sql: string;
-    readonly bind: readonly Value[];
-};
 export type QueryAnswer = QueryRows & {
     readonly more?: string;
 };
-export declare function selected(connection: Connection, plan: Plan, rooms: readonly QueryRoom[]): Fragment;
 export declare function answerQuery(connection: Connection, plan: Plan, rooms: readonly QueryRoom[], budget?: number | null): QueryAnswer;
 //# sourceMappingURL=compile.d.ts.map

package/dist/compile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../src/compile.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAOV,IAAI,EAGJ,SAAS,EAIV,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAE,UAAU,EAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAU1D,MAAM,MAAM,SAAS,GAAG;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3E,MAAM,MAAM,QAAQ,GAAG;IAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,KAAK,EAAE,CAAA;CAAE,CAAC;AAssBjF,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG;IAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA4TjE,wBAAgB,QAAQ,CACtB,UAAU,EAAE,UAAU,EACtB,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,SAAS,SAAS,EAAE,GAC1B,QAAQ,CAEV;AAED,wBAAgB,WAAW,CACzB,UAAU,EAAE,UAAU,EACtB,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,SAAS,SAAS,EAAE,EAC3B,MAAM,GAAE,MAAM,GAAG,IAAW,GAC3B,WAAW,CAUb"}
+{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../src/compile.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,IAAI,EAAuC,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvF,OAAO,KAAK,EAAE,UAAU,EAAc,MAAM,aAAa,CAAC;AAG1D,MAAM,MAAM,SAAS,GAAG;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAkb3E,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG;IAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAkUjE,wBAAgB,WAAW,CACzB,UAAU,EAAE,UAAU,EACtB,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,SAAS,SAAS,EAAE,EAC3B,MAAM,GAAE,MAAM,GAAG,IAAW,GAC3B,WAAW,CAUb"}

package/dist/compile.js

@@ -1,125 +1,36 @@
-import { ARRAY_INDEX, parse, pointerSegments } from '@agentshouse/mdmodel';
+import { ARRAY_INDEX, bodied, byCodePoint, compared, cursorOf, heldKind, parse, pointerSegments, sql, } from '@agentshouse/mdmodel';
 import { binaryRecord } from './binary.js';
-import { documentKey, kindOf, pathOfKey, placedOf, roomOfKey } from './rows.js';
-import { byCodePoint, compared, cursorOf, FROM_START, MAXIMUM_WINDOW, } from './windows.js';
+import { documentKey, pathOfKey, roomOfKey } from './rows.js';
+import { FROM_START, MAXIMUM_WINDOW } from './windows.js';
+const { all, any, exists, fragment, grouped, joined, listed } = sql;
 const VISIBLE = (alias) => `${alias}.visible = 1 AND ${alias}.present = 1 AND ${alias}.queried = 1`;
-function fragment(sql, bind = []) {
-    return { sql, bind };
+function located(alias) {
+    return fragment(`(SELECT k.locator FROM replica_keys k WHERE k.key = ${alias}.key)`);
 }
-function joined(parts, separator) {
-    return {
-        sql: parts.map((part) => part.sql).join(separator),
-        bind: parts.flatMap((part) => part.bind),
-    };
-}
-function balanced(parts, separator) {
-    if (parts.length === 1)
-        return grouped(parts[0]);
-    const half = Math.ceil(parts.length / 2);
-    return grouped(joined([balanced(parts.slice(0, half), separator), balanced(parts.slice(half), separator)], separator));
-}
-function all(parts) {
-    return parts.length === 0 ? fragment('1') : balanced(parts, ' AND ');
-}
-function any(parts) {
-    return parts.length === 0 ? fragment('0') : balanced(parts, ' OR ');
-}
-function grouped(part) {
-    return { sql: `(${part.sql})`, bind: part.bind };
-}
-function list(values) {
-    return fragment(`(${values.map(() => '?').join(', ')})`, values);
-}
-function sqlValue(value) {
-    if (value === null)
-        return { kind: 'null', value: null };
-    if (typeof value === 'boolean')
-        return { kind: 'boolean', value: value ? 1 : 0 };
-    if (typeof value === 'number')
-        return { kind: 'number', value };
-    return { kind: 'string', value };
-}
-function same(alias, value) {
-    const held = sqlValue(value);
-    return held.kind === 'null'
-        ? fragment(`${alias}.kind = 'null'`)
-        : fragment(`${alias}.kind = ? AND ${alias}.value = ?`, [held.kind, held.value]);
-}
-const BOUNDS = { gt: '>', gte: '>=', lt: '<', lte: '<=' };
-function pointed(alias, field, reach) {
-    const { keys, places, glob } = placedOf(field, reach);
-    return fragment(`${alias}.keys = ? AND ${alias}.places ${glob ? 'GLOB' : '='} ?`, [keys, places]);
-}
-function heldValues(doc, field, alias, where) {
-    return joined([
-        fragment(`SELECT 1 FROM replica_values ${alias} WHERE ${alias}.doc = ${doc}.id AND `),
-        grouped(pointed(alias, field, 'value')),
-        fragment(' AND '),
-        grouped(where),
-    ], '');
-}
-function exists(part) {
-    return { sql: `EXISTS (${part.sql})`, bind: part.bind };
-}
-function itemOf(held, where) {
-    return exists(joined([
-        fragment(`SELECT 1 FROM replica_values i WHERE i.doc = ${held}.doc AND i.keys = ${held}.keys AND i.places = ${held}.places AND i.item >= 0 AND `),
-        grouped(where),
-    ], ''));
-}
-function matching(held, values) {
-    return any(values.map((value) => any([
-        same(held, value),
-        all([fragment(`${held}.kind = 'array'`), itemOf(held, same('i', value))]),
-    ])));
-}
-function tested(doc, field, test) {
-    const reached = exists(heldValues(doc, field, 'r', fragment('r.item = -1')));
-    if (test.op === 'exists')
-        return test.exists ? reached : fragment(`NOT ${reached.sql}`, reached.bind);
-    if (test.op === 'eq' || test.op === 'in') {
-        const values = test.op === 'eq' ? [test.value] : test.values;
-        return exists(heldValues(doc, field, 'v', any(values.map((value) => same('v', value)))));
-    }
-    if (test.op === 'ne' || test.op === 'nin') {
-        const values = test.op === 'ne' ? [test.value] : test.values;
-        const unmatched = matching('h', values);
-        return any([
-            fragment(`NOT ${reached.sql}`, reached.bind),
-            exists(heldValues(doc, field, 'h', all([fragment('h.item = -1'), fragment(`NOT (${unmatched.sql})`, unmatched.bind)]))),
-        ]);
-    }
-    if (test.op === 'range') {
-        return exists(heldValues(doc, field, 'v', all([
-            fragment('v.item = -1'),
-            ...test.bounds.map(([operator, bound]) => {
-                const held = sqlValue(bound);
-                return fragment(`v.kind = ? AND v.value ${BOUNDS[operator]} ?`, [
-                    held.kind,
-                    held.value,
-                ]);
-            }),
-        ])));
-    }
-    if (test.op === 'all') {
-        return exists(heldValues(doc, field, 'h', all([
-            fragment("h.item = -1 AND h.kind = 'array'"),
-            ...test.values.map((value) => itemOf('h', same('i', value))),
-        ])));
-    }
-    return exists(heldValues(doc, field, 'v', fragment("v.item = -1 AND v.kind = 'array' AND v.size = ?", [test.size])));
+function within(alias, path) {
+    return fragment(`(${alias}.path = ? OR (${alias}.path >= ? AND ${alias}.path < ?))`, [
+        path,
+        `${path}/`,
+        `${path}0`,
+    ]);
 }
-function conditioned(doc, condition) {
-    if ('every' in condition)
-        return all(condition.every.map((one) => conditioned(doc, one)));
-    if ('some' in condition)
-        return any(condition.some.map((one) => conditioned(doc, one)));
-    return tested(doc, condition.field, condition.test);
+function ancestors(area) {
+    const depth = area.path.split('/').length;
+    return area.open.flatMap((open) => {
+        const segments = open.split('/');
+        return segments.slice(depth - 1, -1).map((_, at) => segments.slice(0, depth + at).join('/'));
+    });
 }
-function located(bound, alias) {
-    if (bound.rooms.length === 0)
-        return fragment("''");
-    return fragment(`CASE ${alias}.key ${bound.rooms.map(() => 'WHEN ? THEN ?').join(' ')} END`, bound.rooms.flatMap(({ key, locator }) => [key, locator]));
+function fenced(alias, key, area, folders) {
+    const opened = any([
+        ...area.open.map((open) => within(alias, open)),
+        ...(folders ? ancestors(area).map((path) => fragment(`${alias}.path = ?`, [path])) : []),
+    ]);
+    return all([
+        fragment(`${alias}.key = ?`, [key]),
+        within(alias, area.path),
+        fragment(`NOT ${grouped(opened).sql}`, opened.bind),
+    ]);
 }
 function reached(bound, alias) {
     const reach = bound.reach;
@@ -133,44 +44,20 @@ function reached(bound, alias) {
         const key = bound.keys.get(roomOfKey(held));
         return key === undefined ? [] : [[key, pathOfKey(held)]];
     });
-    const areas = reach.areas.flatMap((area) => {
+    const outside = any(reach.areas.flatMap((area) => {
         const key = bound.keys.get(area.room);
-        return key === undefined ? [] : [{ key, path: area.path, types: area.types }];
-    });
-    const beneath = (area) => fragment(`${alias}.key = ? AND (${alias}.path = ? OR (${alias}.path >= ? AND ${alias}.path < ?))`, [area.key, area.path, `${area.path}/`, `${area.path}0`]);
-    const typed = (types) => fragment(`${alias}.type IN ${list(types).sql}`, types);
-    const outside = any(areas.map((area) => area.types.length === 0
-        ? beneath(area)
-        : all([beneath(area), fragment(`NOT ${grouped(typed(area.types)).sql}`, area.types)])));
+        return key === undefined ? [] : [fenced(alias, key, area, false)];
+    }));
     return all([
         any([
-            ...(rooms.length === 0 ? [] : [fragment(`${alias}.key IN ${list(rooms).sql}`, rooms)]),
+            ...(rooms.length === 0 ? [] : [fragment(`${alias}.key IN ${listed(rooms).sql}`, rooms)]),
             ...(documents.length === 0
                 ? []
                 : [
                     fragment(`EXISTS (SELECT 1 FROM json_each(?) j WHERE j.value ->> 0 = ${alias}.key AND j.value ->> 1 = ${alias}.path)`, [JSON.stringify(documents)]),
                 ]),
-            ...areas.flatMap((area) => area.types.length === 0 ? [] : [all([beneath(area), typed(area.types)])]),
         ]),
-        ...(areas.length === 0 ? [] : [fragment(`NOT ${grouped(outside).sql}`, outside.bind)]),
-    ]);
-}
-function unindexed(column) {
-    return `+${column}`;
-}
-function scoped(alias, scope, byPaths = false) {
-    const type = byPaths ? unindexed(`${alias}.type`) : `${alias}.type`;
-    return all([
-        ...(scope.types === null ? [] : [fragment(`${type} IN ${list(scope.types).sql}`, scope.types)]),
-        ...(scope.pathPrefix === null || scope.pathPrefix === ''
-            ? []
-            : [
-                fragment(`${alias}.path >= ? AND ${alias}.path < ?`, [
-                    `${scope.pathPrefix}/`,
-                    `${scope.pathPrefix}0`,
-                ]),
-            ]),
-        ...(scope.where === null ? [] : [conditioned(alias, scope.where)]),
+        ...(reach.areas.length === 0 ? [] : [fragment(`NOT ${grouped(outside).sql}`, outside.bind)]),
     ]);
 }
 function related(bound, plan) {
@@ -178,17 +65,17 @@ function related(bound, plan) {
     if (plan.hop !== null) {
         parts.push(exists(joined([
             fragment(`SELECT 1 FROM replica_links l JOIN replica_entries m ON m.key = d.key AND m.path = l.target AND ${VISIBLE('m')} WHERE l.doc = d.id AND l.kind = 'field' AND`),
-            grouped(pointed('l', plan.hop.field, 'link')),
+            grouped(sql.placed('l', plan.hop.field, 'link')),
             fragment('AND'),
-            grouped(all([scoped('m', plan.hop), reached(bound, 'm')])),
+            grouped(all([sql.scoped('m', plan.hop), reached(bound, 'm')])),
         ], ' ')));
     }
     if (plan.backlinks !== null) {
         const field = plan.backlinks.field;
         parts.push(exists(joined([
             fragment(`SELECT 1 FROM replica_links l JOIN replica_entries s ON s.id = l.doc AND ${VISIBLE('s')} WHERE l.key = d.key AND l.target = d.path AND l.kind = 'field' AND`),
-            ...(field === null ? [] : [grouped(pointed('l', field, 'link')), fragment('AND')]),
-            grouped(all([scoped('s', plan.backlinks), reached(bound, 's')])),
+            ...(field === null ? [] : [grouped(sql.placed('l', field, 'link')), fragment('AND')]),
+            grouped(all([sql.scoped('s', plan.backlinks), reached(bound, 's')])),
         ], ' ')));
     }
     return parts;
@@ -196,11 +83,9 @@ function related(bound, plan) {
 function filtered(bound, plan) {
     const keys = bound.rooms.map(({ key }) => key);
     return all([
-        fragment(`d.key IN ${list(keys).sql} AND ${VISIBLE('d')}`, keys),
-        ...(plan.paths === null ? [] : [fragment(`d.path IN ${list(plan.paths).sql}`, plan.paths)]),
-        ...(plan.folder === null ? [] : [fragment('d.folder = ?', [plan.folder])]),
+        fragment(`d.key IN ${listed(keys).sql} AND ${VISIBLE('d')}`, keys),
         reached(bound, 'd'),
-        scoped('d', plan.root, plan.paths !== null),
+        sql.tests('d', plan),
         ...related(bound, plan),
     ]);
 }
@@ -231,103 +116,25 @@ const GROUP_TIES = {
     locale: undefined,
     localeOptions: undefined,
 };
-function classOf(value, ordering) {
-    if (value === null || value === undefined) {
-        return { placement: ordering.nulls === 'first' ? 0 : 2, klass: 0, scalar: null };
-    }
-    if (typeof value === 'boolean')
-        return { placement: 1, klass: 1, scalar: value ? 1 : 0 };
-    if (typeof value === 'number')
-        return { placement: 1, klass: 1, scalar: value };
-    if (typeof value === 'string')
-        return { placement: 1, klass: 2, scalar: value };
-    return { placement: 1, klass: 3, scalar: JSON.stringify(value) };
-}
-function placement(term) {
-    const nulls = term.ordering.nulls === 'first' ? 0 : 2;
-    return fragment(`CASE WHEN ${term.kind.sql} IS NULL OR ${term.kind.sql} = 'null' THEN ${nulls} ELSE 1 END`, [...term.kind.bind, ...term.kind.bind]);
-}
-function klass(term) {
-    return fragment(`CASE ${term.kind.sql} WHEN 'boolean' THEN 1 WHEN 'number' THEN 1 WHEN 'string' THEN 2 WHEN 'array' THEN 3 ELSE 4 END`, term.kind.bind);
-}
-function classed(term, which) {
-    const kinds = which === 1 ? "('boolean', 'number')" : which === 2 ? "('string')" : "('array')";
-    return fragment(`(CASE WHEN ${term.kind.sql} IN ${kinds} THEN ${term.value.sql} END)`, [
-        ...term.kind.bind,
-        ...term.value.bind,
-    ]);
-}
-function collated(bound, term, which) {
-    if (which === 1)
-        return '';
-    const name = bound.collate(term.ordering, which === 3, term.total === true);
-    return name === null ? '' : ` COLLATE ${name}`;
-}
-function sorted(bound, term) {
-    const direction = term.ordering.direction === 'desc' ? 'DESC' : 'ASC';
-    return joined([
-        placement(term),
-        fragment(`${klass(term).sql} ${direction}`, klass(term).bind),
-        ...[1, 2, 3].map((which) => {
-            const value = classed(term, which);
-            return fragment(`${value.sql}${collated(bound, term, which)} ${direction}`, value.bind);
-        }),
-    ], ', ');
-}
-function beyond(bound, term, value) {
-    const held = classOf(value, term.ordering);
-    const place = placement(term);
-    if (held.placement !== 1) {
-        return {
-            beyond: fragment(`${place.sql} > ${held.placement}`, place.bind),
-            equal: fragment(`${place.sql} = ${held.placement}`, place.bind),
-        };
-    }
-    const after = term.ordering.direction === 'desc' ? '<' : '>';
-    const kind = klass(term);
-    const scalar = classed(term, held.klass);
-    const collation = collated(bound, term, held.klass);
-    return {
-        beyond: any([
-            fragment(`${place.sql} > 1`, place.bind),
-            all([
-                fragment(`${place.sql} = 1`, place.bind),
-                any([
-                    fragment(`${kind.sql} ${after} ${held.klass}`, kind.bind),
-                    all([
-                        fragment(`${kind.sql} = ${held.klass}`, kind.bind),
-                        fragment(`${scalar.sql} ${after} ?${collation}`, [...scalar.bind, held.scalar]),
-                    ]),
-                ]),
-            ]),
-        ]),
-        equal: all([
-            fragment(`${place.sql} = 1 AND ${kind.sql} = ${held.klass}`, [...place.bind, ...kind.bind]),
-            fragment(`${scalar.sql} = ?${collation}`, [...scalar.bind, held.scalar]),
-        ]),
-    };
-}
-function following(parts, tie) {
-    let condition = tie;
-    for (const part of [...parts].reverse()) {
-        condition = any([part.beyond, all([part.equal, condition])]);
-    }
-    return condition;
-}
-function keyTerm(bound, key, ordering, index) {
+function keyTerm(key, ordering, index) {
     if (key.of === 'field') {
         const alias = `o${index}`;
         return {
-            term: { ordering, kind: fragment(`${alias}.kind`), value: fragment(`${alias}.value`) },
+            term: {
+                ordering,
+                kind: fragment(`${alias}.kind`),
+                value: fragment(`${alias}.value`),
+                instant: fragment(`${alias}.instant`),
+            },
             join: joined([
                 fragment(`LEFT JOIN replica_values ${alias} ON ${alias}.doc = d.id AND `),
-                grouped(pointed(alias, key.field, 'exact')),
+                grouped(sql.placed(alias, key.field, 'exact')),
                 fragment(` AND ${alias}.item = -1 AND ${alias}.kind NOT IN ('object', 'undefined')`),
             ], ''),
         };
     }
     const expression = key.of === 'room'
-        ? located(bound, 'd')
+        ? located('d')
         : key.of === 'path'
             ? fragment('d.path')
             : key.of === 'type'
@@ -353,13 +160,20 @@ function decoded(kind, value) {
         return JSON.parse(value);
     return null;
 }
+function sortedValue(row, index) {
+    const kind = row[`k${index}`];
+    return sql.sortValue(kind, decoded(kind, row[`v${index}`]), row[`i${index}`]);
+}
 function decodedKey(key, row, index) {
     if (key.of === 'field') {
         const kind = row[`k${index}`];
-        return kind === null || kind === undefined ? undefined : decoded(kind, row[`v${index}`]);
+        return kind === null || kind === undefined ? undefined : sortedValue(row, index);
     }
     return row[`v${index}`] ?? null;
 }
+function instantOf(term) {
+    return term.instant ?? fragment('NULL');
+}
 const registered = new WeakMap();
 function hashed(text) {
     let hash = 0x811c9dc5;
@@ -409,9 +223,6 @@ function collating(connection) {
         return name;
     };
 }
-function bodied(shape) {
-    return shape.sections !== null || shape.preamble || shape.markdown;
-}
 function masked(pointers) {
     const root = { whole: false, members: new Map() };
     for (const pointer of pointers) {
@@ -468,9 +279,10 @@ function documentOf(shape, room, row, linked) {
     const type = row.type;
     const fileId = row.file_id ?? null;
     const revision = row.revision ?? null;
+    const content = row.content ?? null;
+    const unheld = Number(row.unheld) === 1 ? { textHeld: false } : {};
     if (type === null) {
         const binary = row.binary === null ? null : JSON.parse(row.binary);
-        const content = row.content ?? null;
         return {
             room,
             path,
@@ -480,11 +292,12 @@ function documentOf(shape, room, row, linked) {
                     binaryFile: binaryRecord(path, binary, revision ?? undefined),
                 }),
             ...(shape.markdown && content !== null ? { markdown: content } : {}),
+            ...unheld,
             ...(shape.fileId ? { fileId } : {}),
             ...(shape.revision ? { revision } : {}),
         };
     }
-    const read = bodied(shape) ? parse(row.content, path) : null;
+    const read = bodied(shape) && content !== null ? parse(content, path) : null;
     const body = read?.ok === true ? read.document : null;
     const fields = JSON.parse(row.fields ?? '{}');
     return {
@@ -502,7 +315,8 @@ function documentOf(shape, room, row, linked) {
             ? {}
             : { sections: sectionsOf(body.sections, shape.sections) }),
         ...(shape.preamble && body !== null ? { preamble: body.preamble } : {}),
-        ...(shape.markdown && body !== null ? { markdown: body.content } : {}),
+        ...(shape.markdown && content !== null ? { markdown: content } : {}),
+        ...unheld,
         ...(shape.backlinks ? { backlinks: linked.get(documentKey(room, path)) ?? [] } : {}),
         ...(shape.change
             ? {
@@ -536,10 +350,10 @@ function backlinksOf(connection, bound, rows) {
     return linked;
 }
 function documents(connection, bound, plan, shape, budget) {
-    const terms = shape.orderBy.map(({ key, options }, index) => keyTerm(bound, key, orderingOf(options), index));
+    const terms = shape.orderBy.map(({ key, options }, index) => keyTerm(key, orderingOf(options), index));
     const filter = filtered(bound, plan);
     const cursor = plan.after?.values ?? null;
-    const room = located(bound, 'd');
+    const room = located('d');
     const tie = cursor === null
         ? null
         : (() => {
@@ -555,19 +369,20 @@ function documents(connection, bound, plan, shape, budget) {
         })();
     const after = cursor === null || tie === null
         ? null
-        : following(terms.map(({ term }, index) => beyond(bound, term, cursor[index])), tie);
+        : sql.following(terms.map(({ term }, index) => sql.beyond(bound.collate, term, cursor[index])), tie);
     const selected = joined([
-        fragment('SELECT d.id, d.key, d.path, d.type, d.binary, d.revision, d.fields, d.change, d.file_id'),
+        fragment('SELECT d.id, d.key, d.path, d.type, d.binary, d.revision, d.fields, d.change, d.file_id, d.content IS NULL AND d.outline IS NOT NULL AS unheld'),
         ...terms.map(({ term }, index) => joined([
             fragment(`, ${term.kind.sql} AS k${index}`, term.kind.bind),
             fragment(`, ${term.value.sql} AS v${index}`, term.value.bind),
+            fragment(`, ${instantOf(term).sql} AS i${index}`, instantOf(term).bind),
         ], '')),
         fragment(' FROM replica_entries d'),
         ...terms.flatMap(({ join }) => (join === null ? [] : [fragment(' '), join])),
         fragment(' WHERE '),
         after === null ? filter : all([filter, after]),
         fragment(' ORDER BY '),
-        joined([...terms.map(({ term }) => sorted(bound, term)), room, fragment('d.path')], ', '),
+        joined([...terms.map(({ term }) => sql.sorted(bound.collate, term)), room, fragment('d.path')], ', '),
         ...(plan.limit === null ? [] : [fragment(' LIMIT ?', [plan.limit + 1])]),
     ], '');
     const rows = connection.rows(selected.sql, selected.bind);
@@ -606,7 +421,7 @@ function withContent(connection, rows, budget) {
         return rows;
     const ids = rows.map((row) => row.id);
     const sizes = new Map(connection
-        .rows(`SELECT id, octet_length(content) AS size FROM replica_entries WHERE id IN ${list(ids).sql}`, ids)
+        .rows(`SELECT id, octet_length(content) AS size FROM replica_entries WHERE id IN ${listed(ids).sql}`, ids)
         .map((row) => [row.id, Number(row.size ?? 0)]));
     let cut = rows.length;
     let spent = 0;
@@ -621,34 +436,30 @@ function withContent(connection, rows, budget) {
     const kept = rows.slice(0, cut);
     const read = kept.map((row) => row.id);
     const held = new Map(connection
-        .rows(`SELECT id, content FROM replica_entries WHERE id IN ${list(read).sql}`, read)
+        .rows(`SELECT id, content FROM replica_entries WHERE id IN ${listed(read).sql}`, read)
         .map((row) => [row.id, row.content]));
     return kept.map((row) => ({ ...row, content: held.get(row.id) ?? null }));
 }
 function storedFolders(bound, plan) {
-    if (plan.paths !== null || plan.hop !== null || plan.backlinks !== null)
+    if (plan.paths !== null || plan.fileIds !== null)
+        return null;
+    if (plan.hop !== null || plan.backlinks !== null)
         return null;
     const whole = bound.rooms.filter(({ locator }) => bound.reach === null || bound.reach.rooms.includes(locator));
     if (whole.length === 0)
         return null;
     const keys = whole.map(({ key }) => key);
     const prefix = plan.root.pathPrefix;
-    const fenced = (bound.reach?.areas ?? []).flatMap((area) => {
+    const outside = (bound.reach?.areas ?? []).flatMap((area) => {
         const key = bound.keys.get(area.room);
-        return key === undefined
-            ? []
-            : [
-                fragment('NOT (f.key = ? AND (f.path = ? OR (f.path >= ? AND f.path < ?)))', [
-                    key,
-                    area.path,
-                    `${area.path}/`,
-                    `${area.path}0`,
-                ]),
-            ];
+        if (key === undefined)
+            return [];
+        const fence = fenced('f', key, area, true);
+        return [fragment(`NOT ${grouped(fence).sql}`, fence.bind)];
     });
     return all([
-        fragment(`f.key IN ${list(keys).sql} AND f.layer = 0 AND f.present = 1`, keys),
-        ...fenced,
+        fragment(`f.key IN ${listed(keys).sql} AND f.layer = 0 AND f.present = 1`, keys),
+        ...outside,
         ...(prefix === null || prefix === ''
             ? []
             : [
@@ -663,7 +474,7 @@ function storedFolders(bound, plan) {
 }
 function groups(connection, bound, plan, shape) {
     const keyed = shape.groupBy.map((key, index) => {
-        const { term, join } = keyTerm(bound, key, GROUP_ORDER, index);
+        const { term, join } = keyTerm(key, GROUP_ORDER, index);
         const kind = fragment(`COALESCE(${term.kind.sql}, 'null')`, term.kind.bind);
         return { term: { ...term, kind }, join };
     });
@@ -673,6 +484,7 @@ function groups(connection, bound, plan, shape) {
         joined(keyed.map(({ term }, index) => joined([
             fragment(`${term.kind.sql} AS k${index}`, term.kind.bind),
             fragment(`, ${term.value.sql} AS v${index}`, term.value.bind),
+            fragment(`, ${instantOf(term).sql} AS i${index}`, instantOf(term).bind),
         ], '')), ', '),
         fragment(', 1 AS n FROM replica_entries d'),
         ...keyed.flatMap(({ join }) => (join === null ? [] : [fragment(' '), join])),
@@ -686,10 +498,15 @@ function groups(connection, bound, plan, shape) {
         ? inner
         : joined([
             inner,
-            fragment(" UNION ALL SELECT 'string' AS k0, f.path AS v0, 0 AS n FROM replica_folders f WHERE "),
+            fragment(" UNION ALL SELECT 'string' AS k0, f.path AS v0, NULL AS i0, 0 AS n FROM replica_folders f WHERE "),
             folders,
         ], '');
-    const columns = keyed.map((_, index) => `k${index}, v${index}`).join(', ');
+    const columns = keyed
+        .map((_, index) => `k${index}, min(v${index}) AS v${index}, i${index}`)
+        .join(', ');
+    const held = keyed
+        .map((_, index) => `k${index}, i${index}, CASE WHEN i${index} IS NULL THEN v${index} END`)
+        .join(', ');
     const counted = {
         ordering: { ...GROUP_ORDER },
         kind: fragment("'number'"),
@@ -702,7 +519,7 @@ function groups(connection, bound, plan, shape) {
     });
     const ordered = shape.orderBy.map(({ by, direction }) => by === 'count'
         ? { ...counted, ordering: { ...GROUP_ORDER, direction } }
-        : outer(by, { ...GROUP_ORDER, direction }));
+        : { ...outer(by, { ...GROUP_ORDER, direction }), instant: fragment(`i${by}`) });
     const ties = keyed.flatMap((_, index) => [
         { ...outer(index, GROUP_TIES), total: true },
         { ordering: GROUP_TIES, kind: fragment("'string'"), value: fragment(`k${index}`) },
@@ -712,19 +529,21 @@ function groups(connection, bound, plan, shape) {
         ? null
         : cursor[shape.orderBy.length].flatMap((value) => [
             value,
-            kindOf(value ?? null),
+            heldKind(value ?? null),
         ]);
     const after = cursor === null || tieValues === null
         ? null
-        : following(ordered.map((term, index) => beyond(bound, term, cursor[index])), following(ties.slice(0, -1).map((term, index) => beyond(bound, term, tieValues[index])), beyond(bound, ties.at(-1), tieValues.at(-1)).beyond));
-    const grouping = joined([fragment(`SELECT ${columns}, sum(n) AS n FROM (`), source, fragment(`) GROUP BY ${columns}`)], '');
+        : sql.following(ordered.map((term, index) => sql.beyond(bound.collate, term, cursor[index])), sql.following(ties
+            .slice(0, -1)
+            .map((term, index) => sql.beyond(bound.collate, term, tieValues[index])), sql.beyond(bound.collate, ties.at(-1), tieValues.at(-1)).beyond));
+    const grouping = joined([fragment(`SELECT ${columns}, sum(n) AS n FROM (`), source, fragment(`) GROUP BY ${held}`)], '');
     const windowed = joined([
         fragment('SELECT * FROM ('),
         grouping,
         fragment(')'),
         ...(after === null ? [] : [fragment(' WHERE '), after]),
         fragment(' ORDER BY '),
-        joined([...ordered, ...ties].map((term) => sorted(bound, term)), ', '),
+        joined([...ordered, ...ties].map((term) => sql.sorted(bound.collate, term)), ', '),
         ...(plan.limit === null ? [] : [fragment(' LIMIT ?', [plan.limit + 1])]),
     ], '');
     const rows = connection.rows(windowed.sql, windowed.bind);
@@ -737,7 +556,7 @@ function groups(connection, bound, plan, shape) {
         : last === undefined
             ? (plan.after?.text ?? FROM_START)
             : cursorOf([
-                ...shape.orderBy.map(({ by }) => by === 'count' ? Number(last.n) : decoded(last[`k${by}`], last[`v${by}`])),
+                ...shape.orderBy.map(({ by }) => by === 'count' ? Number(last.n) : sortedValue(last, by)),
                 keyOf(last),
             ]);
     const total = plan.total
@@ -757,9 +576,6 @@ function boundOf(connection, plan, rooms) {
         collate: collating(connection),
     };
 }
-export function selected(connection, plan, rooms) {
-    return filtered(boundOf(connection, plan, rooms), plan);
-}
 export function answerQuery(connection, plan, rooms, budget = null) {
     const bound = boundOf(connection, plan, rooms);
     if (rooms.length === 0) {