@coffer-org/server 3.4.0 → 5.0.0

package/dist/auth-api.js

@@ -1,13 +1,14 @@
 import { field } from '@coffer-org/sdk/fields';
+import { materializeTree } from '@coffer-org/sdk/materialize/pipeline';
 import { countUsers, createUser, findUserByLogin, findUserById, getPasswordHash, listUsers, updateUser, deleteUser, countAdmins, createSession, resolveSession, deleteSession, createApiToken, resolveApiToken, listApiTokens, revokeApiToken, } from "./auth-store.js";
 import { resolveAccessToken } from "./oauth-store.js";
 import { mcpResource } from "./public-url.js";
 import { verifyPassword } from "./auth-crypto.js";
 import { maskSecrets, preserveSecrets } from "./field-masking.js";
 import { rowMatch } from "./records-api.js";
-import { tokenize } from '@coffer-org/core/search';
+import { tokenize } from '@coffer-org/sdk/search';
 import { USERS_SHELF } from '@coffer-org/sdk/users-shelf';
-const PASSWORD_FIELDS = [field.password({ key: 'password' })];
+const PASSWORD_FIELDS = materializeTree({ password: field.password({}) });
 const SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000;
 const COOKIE_NAME = 'sid';
 export const PUBLIC_API_PATHS = ['/api/auth/status', '/api/auth/login', '/api/auth/setup'];
@@ -75,9 +76,7 @@ export async function registerAuthApi(app) {
             return reply.code(409).send({ error: 'already_setup' });
         const body = (req.body ?? {});
         if (!body.login || !body.password || !body.displayName) {
-            return reply
-                .code(422)
-                .send({
+            return reply.code(422).send({
                 issues: [{ field: !body.login ? 'login' : !body.password ? 'password' : 'displayName', code: 'required' }],
             });
         }

package/dist/cache-clock.d.ts

@@ -0,0 +1,8 @@
+import type { LayoutEl } from '@coffer-org/sdk/fields';
+export declare function wholeClockColumn(key: string): string;
+export declare function partClockColumn(key: string, partKey: string): string;
+export declare function cacheClockColumns(fields: LayoutEl[]): string[];
+export declare function levelClockColumnsByOwner(fields: LayoutEl[]): Map<string, string[]>;
+export declare function stripClockKeys(row: Record<string, unknown>): Record<string, unknown>;
+export declare function stripClockKeysDeep(fields: LayoutEl[], row: Record<string, unknown>): Record<string, unknown>;
+export declare function clockValuesOf(row: Record<string, unknown>): Record<string, number>;

package/dist/cache-clock.js

@@ -0,0 +1,103 @@
+import { fieldEntries } from '@coffer-org/sdk/shelf';
+import { hasChildren, isNamedField, isCollectionGroup, isEmbeddedGroup, isStorageGroup } from '@coffer-org/sdk/fields';
+export function wholeClockColumn(key) {
+    return `${key}__cachedAt`;
+}
+export function partClockColumn(key, partKey) {
+    return `${key}__${partKey}__cachedAt`;
+}
+export function cacheClockColumns(fields) {
+    const out = [];
+    for (const [key, fm] of fieldEntries(fields)) {
+        if (fm.cache !== undefined)
+            out.push(wholeClockColumn(key));
+        for (const p of fm.parts ?? []) {
+            if (p.key != null && p.meta?.cache !== undefined)
+                out.push(partClockColumn(key, p.key));
+        }
+    }
+    return out;
+}
+export function levelClockColumnsByOwner(fields) {
+    const out = new Map();
+    const add = (owner, col) => {
+        const list = out.get(owner);
+        if (list)
+            list.push(col);
+        else
+            out.set(owner, [col]);
+    };
+    const scan = (list) => {
+        for (const it of list) {
+            if (isNamedField(it)) {
+                const fm = it.type;
+                if (fm.cache !== undefined)
+                    add(it.key, wholeClockColumn(it.key));
+                for (const p of fm.parts ?? []) {
+                    if (p.key != null && p.meta?.cache !== undefined)
+                        add(it.key, partClockColumn(it.key, p.key));
+                }
+            }
+            else if (hasChildren(it) && it.scope === 'hoist') {
+                scan(it.fields);
+            }
+        }
+    };
+    scan(fields);
+    return out;
+}
+function levelClockColumns(fields) {
+    return [...levelClockColumnsByOwner(fields).values()].flat();
+}
+export function stripClockKeys(row) {
+    for (const k of Object.keys(row))
+        if (k.endsWith('__cachedAt'))
+            delete row[k];
+    return row;
+}
+function levelGroups(fields) {
+    const embedded = [];
+    const collections = [];
+    const scan = (list) => {
+        for (const it of list) {
+            if (!hasChildren(it))
+                continue;
+            if (!isStorageGroup(it))
+                scan(it.fields);
+            else if (isCollectionGroup(it))
+                collections.push([it.key, it.fields]);
+            else if (isEmbeddedGroup(it))
+                embedded.push([it.key, it.fields]);
+        }
+    };
+    scan(fields);
+    return { embedded, collections };
+}
+export function stripClockKeysDeep(fields, row) {
+    for (const col of levelClockColumns(fields))
+        delete row[col];
+    const { embedded, collections } = levelGroups(fields);
+    for (const [key, sub] of embedded) {
+        const nested = row[key];
+        if (nested != null && typeof nested === 'object' && !Array.isArray(nested)) {
+            stripClockKeysDeep(sub, nested);
+        }
+    }
+    for (const [key, sub] of collections) {
+        const rows = row[key];
+        if (Array.isArray(rows)) {
+            for (const r of rows)
+                if (r != null && typeof r === 'object')
+                    stripClockKeysDeep(sub, r);
+        }
+    }
+    return row;
+}
+export function clockValuesOf(row) {
+    const out = {};
+    for (const [k, v] of Object.entries(row)) {
+        if (k.endsWith('__cachedAt') && typeof v === 'number')
+            out[k] = v;
+    }
+    return out;
+}

package/dist/cache-refresh.d.ts

@@ -0,0 +1,7 @@
+import type { ShelfDef } from '@coffer-org/sdk/shelf';
+import { type RowMeta } from './collection-io.ts';
+import { type PartCtx } from './part-injection.ts';
+type Row = Record<string, unknown>;
+export declare function stale(cache: number, cachedAt: number | null | undefined, now: number): boolean;
+export declare function refreshCachedUnits(m: ShelfDef, tableName: string, row: Row, clocks: Record<string, number>, collectionMeta: Record<string, RowMeta[]>, ctx: PartCtx): Promise<Row>;
+export {};

package/dist/cache-refresh.js

@@ -0,0 +1,322 @@
+import { fieldEntries, storageColumns, collectionGroups } from '@coffer-org/sdk/shelf';
+import { hasChildren, isNamedField, isCollectionGroup, isStorageGroup, isJsonStored, } from '@coffer-org/sdk/fields';
+import { getEm } from "./db.js";
+import { flattenEmbeddedAt } from "./collection-io.js";
+import { encodeTemporalAt } from "./temporal.js";
+import { wholeClockColumn, partClockColumn, cacheClockColumns } from "./cache-clock.js";
+import { sameValue } from "./value-compare.js";
+import { runTriggers } from "./trigger-queue.js";
+import { getActiveRegistry } from "./registry-context.js";
+export function stale(cache, cachedAt, now) {
+    return cache === 0 || cachedAt == null || now - cachedAt >= cache * 1000;
+}
+function collectCacheUnits(fields, scopePath, flatPrefix) {
+    const out = [];
+    for (const it of fields) {
+        if (isNamedField(it)) {
+            const fm = it.type;
+            const fullKey = `${flatPrefix}${it.key}`;
+            if (fm.cache !== undefined) {
+                out.push({
+                    cache: fm.cache,
+                    clockColumn: wholeClockColumn(fullKey),
+                    valueColumns: storageColumns(fullKey, fm).map(([col]) => col),
+                    scopePath,
+                    fieldKey: it.key,
+                });
+            }
+            const allPartKeys = (fm.parts ?? []).filter((p) => p.key != null).map((p) => p.key);
+            for (const p of fm.parts ?? []) {
+                if (p.key != null && p.meta?.cache !== undefined) {
+                    out.push({
+                        cache: p.meta.cache,
+                        clockColumn: partClockColumn(fullKey, p.key),
+                        valueColumns: [`${fullKey}__${p.key}`],
+                        scopePath,
+                        fieldKey: it.key,
+                        role: p.key,
+                        allPartKeys,
+                    });
+                }
+            }
+        }
+        else if (hasChildren(it) && !isCollectionGroup(it)) {
+            const sub = it.fields;
+            if (!isStorageGroup(it))
+                out.push(...collectCacheUnits(sub, scopePath, flatPrefix));
+            else
+                out.push(...collectCacheUnits(sub, [...scopePath, it.key], `${flatPrefix}${it.key}__`));
+        }
+    }
+    return out;
+}
+function collectCollectionGroups(fields, tablePrefix) {
+    return collectionGroups(fields).map((c) => {
+        const table = `${tablePrefix}__${c.key}`;
+        const childFields = c.group.fields;
+        return {
+            key: c.key,
+            table,
+            fields: childFields,
+            units: collectCacheUnits(childFields, [], ''),
+            children: collectCollectionGroups(childFields, table),
+        };
+    });
+}
+function anyCollectionWork(groups) {
+    return groups.some((g) => g.units.length > 0 || anyCollectionWork(g.children));
+}
+const infoCache = new WeakMap();
+function shelfCacheInfoOf(m) {
+    let info = infoCache.get(m.fields);
+    if (!info) {
+        info = {
+            base: collectCacheUnits(m.fields, [], ''),
+            collections: collectCollectionGroups(m.fields, `${m.library}__${m.shelf}`),
+        };
+        infoCache.set(m.fields, info);
+    }
+    return info;
+}
+function isPlainRow(v) {
+    return v != null && typeof v === 'object' && !Array.isArray(v);
+}
+function scopeAt(obj, path) {
+    let cur = obj;
+    for (const seg of path) {
+        if (cur == null)
+            return undefined;
+        if (typeof seg === 'number') {
+            if (!Array.isArray(cur))
+                return undefined;
+            cur = cur[seg];
+        }
+        else {
+            if (typeof cur !== 'object' || Array.isArray(cur))
+                return undefined;
+            cur = cur[seg];
+        }
+    }
+    return cur;
+}
+function readUnitValue(u, obj, groupChain = []) {
+    const scope = scopeAt(obj, [...groupChain, ...u.scopePath]);
+    if (!scope)
+        return undefined;
+    if (u.role === undefined)
+        return scope[u.fieldKey];
+    const composite = scope[u.fieldKey];
+    return composite != null && typeof composite === 'object' ? composite[u.role] : undefined;
+}
+function readUnitClock(u, obj, groupChain = []) {
+    const scope = scopeAt(obj, [...groupChain, ...u.scopePath]);
+    if (!scope)
+        return undefined;
+    const localClockKey = u.role === undefined ? wholeClockColumn(u.fieldKey) : partClockColumn(u.fieldKey, u.role);
+    const v = scope[localClockKey];
+    return typeof v === 'number' ? v : undefined;
+}
+function withValueAt(container, path, key, value) {
+    const [head, ...rest] = path;
+    if (head === undefined) {
+        const base = isPlainRow(container) ? container : {};
+        return { ...base, [key]: value };
+    }
+    if (typeof head === 'number') {
+        const arr = Array.isArray(container) ? container : [];
+        const next = arr.slice();
+        next[head] = withValueAt(arr[head], rest, key, value);
+        return next;
+    }
+    const obj = isPlainRow(container) ? container : {};
+    return { ...obj, [head]: withValueAt(obj[head], rest, key, value) };
+}
+function withUnitValue(row, groupChain, u, value) {
+    const path = [...groupChain, ...u.scopePath];
+    if (u.role === undefined)
+        return withValueAt(row, path, u.fieldKey, value);
+    const scope = scopeAt(row, path);
+    const existing = scope ? scope[u.fieldKey] : undefined;
+    const composite = isPlainRow(existing)
+        ? { ...existing }
+        : Object.fromEntries((u.allPartKeys ?? [u.role]).map((k) => [k, null]));
+    composite[u.role] = value;
+    return withValueAt(row, path, u.fieldKey, composite);
+}
+function encodeJsonForColumns(fields, row) {
+    const out = { ...row };
+    for (const [k, f] of fieldEntries(fields)) {
+        if (isJsonStored(f) && out[k] !== null && out[k] !== undefined && typeof out[k] === 'object') {
+            out[k] = JSON.stringify(out[k]);
+        }
+    }
+    return out;
+}
+function flatColumnsOf(fields, obj) {
+    return encodeJsonForColumns(fields, encodeTemporalAt(fields, flattenEmbeddedAt(fields, obj)));
+}
+function unitValueSame(u, before, after) {
+    return u.valueColumns.every((col) => sameValue(before[col] ?? null, after[col] ?? null));
+}
+function nowIso() {
+    return new Date().toISOString();
+}
+function planCollectionStale(groups, record, meta, prefixChain, now, out) {
+    for (const g of groups) {
+        const rows = record[g.key] ?? [];
+        const metas = meta[g.key] ?? [];
+        rows.forEach((rowObj, i) => {
+            const rm = metas[i];
+            if (!rm)
+                return;
+            const chain = [...prefixChain, g.key, i];
+            for (const u of g.units) {
+                const prevClock = rm.clocks[u.clockColumn] ?? null;
+                if (stale(u.cache, prevClock, now)) {
+                    out.push({ table: g.table, rowId: rm.id, fields: g.fields, groupChain: chain, unit: u, prevClock });
+                }
+            }
+            if (g.children.length && isPlainRow(rowObj)) {
+                planCollectionStale(g.children, rowObj, rm.nested, chain, now, out);
+            }
+        });
+    }
+}
+function groupByRow(entries) {
+    const out = new Map();
+    for (const e of entries) {
+        const k = `${e.table}#${e.rowId}`;
+        let g = out.get(k);
+        if (!g) {
+            g = { table: e.table, rowId: e.rowId, fields: e.fields, groupChain: e.groupChain, entries: [] };
+            out.set(k, g);
+        }
+        g.entries.push(e);
+    }
+    return out;
+}
+function cacheUnitPath(u) {
+    return u.role === undefined ? [...u.scopePath, u.fieldKey] : [...u.scopePath, u.fieldKey, u.role];
+}
+function chainKeys(chain) {
+    return chain.filter((s) => typeof s === 'string');
+}
+export async function refreshCachedUnits(m, tableName, row, clocks, collectionMeta, ctx) {
+    const info = shelfCacheInfoOf(m);
+    if (info.base.length === 0 && !anyCollectionWork(info.collections))
+        return row;
+    const now = Date.now();
+    const baseStale = info.base.filter((u) => stale(u.cache, clocks[u.clockColumn] ?? null, now));
+    const collectionStale = [];
+    planCollectionStale(info.collections, row, collectionMeta, [], now, collectionStale);
+    if (baseStale.length === 0 && collectionStale.length === 0)
+        return row;
+    const graph = getActiveRegistry().mutateGraph;
+    const shelfKey = `${m.library}/${m.shelf}`;
+    const seedTargets = [
+        ...baseStale.map((u) => ({ hops: [], path: cacheUnitPath(u), relative: false })),
+        ...collectionStale.map((e) => ({
+            hops: [],
+            path: [...chainKeys(e.groupChain), ...cacheUnitPath(e.unit)],
+            relative: false,
+        })),
+    ];
+    const recomputed = await runTriggers(m.fields, row, [], graph, shelfKey, ctx, { seedTargets });
+    const baseFinished = baseStale.filter((u) => readUnitClock(u, recomputed) !== undefined);
+    let baseChanged = [];
+    let flatRecomputedBase;
+    if (baseFinished.length > 0) {
+        flatRecomputedBase = flatColumnsOf(m.fields, recomputed);
+        const flatBeforeBase = flatColumnsOf(m.fields, row);
+        baseChanged = baseFinished.filter((u) => !unitValueSame(u, flatBeforeBase, flatRecomputedBase));
+    }
+    const collectionWrites = [];
+    for (const group of groupByRow(collectionStale).values()) {
+        const rowRecomputed = scopeAt(recomputed, group.groupChain);
+        if (!rowRecomputed)
+            continue;
+        const finished = group.entries.filter((e) => readUnitClock(e.unit, rowRecomputed) !== undefined);
+        if (finished.length === 0)
+            continue;
+        const rowBefore = scopeAt(row, group.groupChain);
+        if (!rowBefore)
+            continue;
+        const flatRow = flatColumnsOf(group.fields, rowRecomputed);
+        const flatBeforeRow = flatColumnsOf(group.fields, rowBefore);
+        const changed = new Set(finished.filter((e) => !unitValueSame(e.unit, flatBeforeRow, flatRow)));
+        collectionWrites.push({ group, finished, changed, flatRow });
+    }
+    if (baseFinished.length === 0 && collectionWrites.length === 0)
+        return row;
+    let out = row;
+    const ts = nowIso();
+    const persist = async (em) => {
+        if (baseFinished.length > 0) {
+            const dbPatch = {};
+            for (const u of baseFinished)
+                dbPatch[u.clockColumn] = readUnitClock(u, recomputed);
+            for (const u of baseChanged)
+                for (const col of u.valueColumns)
+                    dbPatch[col] = flatRecomputedBase[col] ?? null;
+            if (baseChanged.length > 0)
+                dbPatch['updated_at'] = ts;
+            const id = row['id'];
+            const affected = await em.nativeUpdate(tableName, { id, updated_at: row['updated_at'] }, dbPatch);
+            if (affected > 0) {
+                for (const u of baseFinished)
+                    out = withUnitValue(out, [], u, readUnitValue(u, recomputed));
+                if (baseChanged.length > 0)
+                    out = { ...out, updated_at: ts };
+            }
+        }
+        for (const { group, finished, changed, flatRow } of collectionWrites) {
+            const rowRecomputed = scopeAt(recomputed, group.groupChain);
+            const dbPatch = {};
+            const where = { id: group.rowId };
+            for (const e of finished) {
+                dbPatch[e.unit.clockColumn] = readUnitClock(e.unit, rowRecomputed);
+                where[e.unit.clockColumn] = e.prevClock;
+                if (changed.has(e))
+                    for (const col of e.unit.valueColumns)
+                        dbPatch[col] = flatRow[col] ?? null;
+            }
+            const affected = await em.nativeUpdate(group.table, where, dbPatch);
+            if (affected > 0) {
+                for (const e of finished) {
+                    out = withUnitValue(out, group.groupChain, e.unit, readUnitValue(e.unit, rowRecomputed));
+                }
+            }
+        }
+        const changed = [
+            ...baseChanged.map((u) => ({ path: cacheUnitPath(u) })),
+            ...collectionWrites.flatMap(({ group, changed: rowChanged }) => [...rowChanged].map((e) => ({ path: [...chainKeys(group.groupChain), ...cacheUnitPath(e.unit)] }))),
+        ];
+        if (changed.length > 0) {
+            const triggered = await runTriggers(m.fields, recomputed, changed, graph, shelfKey, { ...ctx, em }, {
+                seedTargets: [],
+            });
+            const clockCols = new Set(cacheClockColumns(m.fields));
+            const collKeys = new Set(collectionGroups(m.fields).map((c) => c.key));
+            const beforeFlat = flatColumnsOf(m.fields, recomputed);
+            const afterFlat = flatColumnsOf(m.fields, triggered);
+            const cascadePatch = {};
+            for (const [k, v] of Object.entries(afterFlat)) {
+                if (collKeys.has(k) || clockCols.has(k))
+                    continue;
+                if (!sameValue(beforeFlat[k] ?? null, v ?? null))
+                    cascadePatch[k] = v ?? null;
+            }
+            if (Object.keys(cascadePatch).length > 0) {
+                const id = row['id'];
+                await em.nativeUpdate(tableName, { id }, cascadePatch);
+                for (const k of Object.keys(cascadePatch))
+                    out = { ...out, [k]: triggered[k] };
+            }
+        }
+    };
+    if (ctx.em)
+        await persist(ctx.em);
+    else
+        await getEm().fork().transactional(persist);
+    return out;
+}

package/dist/collection-io.d.ts

@@ -9,6 +9,15 @@ export declare function splitAt(fields: LayoutEl[], input: Row): {
 };
 export declare function writeAt(tx: EntityManager, prefix: string, fields: LayoutEl[], parentId: number, collections: Record<string, Row[]>): Promise<void>;
 export declare function readAt(em: EntityManager, prefix: string, fields: LayoutEl[], parentId: number): Promise<Record<string, Row[]>>;
+export interface RowMeta {
+    id: number;
+    clocks: Record<string, number>;
+    nested: Record<string, RowMeta[]>;
+}
+export declare function readAtWithMeta(em: EntityManager, prefix: string, fields: LayoutEl[], parentId: number): Promise<{
+    collections: Record<string, Row[]>;
+    meta: Record<string, RowMeta[]>;
+}>;
 export declare function readAtMany(em: EntityManager, prefix: string, fields: LayoutEl[], parentIds: number[]): Promise<Map<number, Record<string, Row[]>>>;
 export declare function deleteAt(tx: EntityManager, prefix: string, fields: LayoutEl[], parentId: number): Promise<void>;
 export declare const splitCollections: (m: ShelfDef, input: Row) => {
@@ -17,6 +26,10 @@ export declare const splitCollections: (m: ShelfDef, input: Row) => {
 };
 export declare const writeCollections: (tx: EntityManager, m: ShelfDef, parentId: number, collections: Record<string, Row[]>) => Promise<void>;
 export declare const readCollections: (em: EntityManager, m: ShelfDef, parentId: number) => Promise<Record<string, Row[]>>;
+export declare const readCollectionsWithMeta: (em: EntityManager, m: ShelfDef, parentId: number) => Promise<{
+    collections: Record<string, Row[]>;
+    meta: Record<string, RowMeta[]>;
+}>;
 export declare const deleteCollections: (tx: EntityManager, m: ShelfDef, parentId: number) => Promise<void>;
 export declare function flattenEmbeddedAt(fields: LayoutEl[], input: Row): Row;
 export declare function nestEmbeddedAt(fields: LayoutEl[], row: Row): Row;

package/dist/collection-io.js

@@ -1,12 +1,13 @@
 import { serialize } from '@mikro-orm/core';
-import { collectionGroups, fieldEntries, storageColumns } from '@coffer-org/sdk/shelf';
-import { isGroup, isEmbeddedGroup, isJsonStored } from '@coffer-org/sdk/fields';
+import { collectionGroups, fieldEntries, flattenComposite, nestComposite, storageColumns } from '@coffer-org/sdk/shelf';
+import { hasChildren, isEmbeddedGroup, isJsonStored } from '@coffer-org/sdk/fields';
 import { chunk } from "./batch.js";
 import { selectRows } from "./read-rows.js";
+import { cacheClockColumns, clockValuesOf } from "./cache-clock.js";
 function flattenLayout(fields) {
     const out = [];
     for (const f of fields) {
-        if (isGroup(f) && !f.key)
+        if (hasChildren(f) && f.scope === 'hoist')
             out.push(...flattenLayout(f.fields));
         else
             out.push(f);
@@ -17,9 +18,10 @@ function collKeySet(fields) {
     return new Set(collectionGroups(flattenLayout(fields)).map((c) => c.key));
 }
 export function scalarKeys(fields) {
-    return fieldEntries(fields)
-        .filter(([, fm]) => !fm.virtual)
-        .flatMap(([k, fm]) => storageColumns(k, fm).map(([c]) => c));
+    return [
+        ...fieldEntries(fields).flatMap(([k, fm]) => storageColumns(k, fm).map(([c]) => c)),
+        ...cacheClockColumns(fields),
+    ];
 }
 export function splitAt(fields, input) {
     const keys = collKeySet(fields);
@@ -83,6 +85,26 @@ export async function readAt(em, prefix, fields, parentId) {
     }
     return out;
 }
+export async function readAtWithMeta(em, prefix, fields, parentId) {
+    const collections = {};
+    const meta = {};
+    for (const c of collectionGroups(fields)) {
+        const table = `${prefix}__${c.key}`;
+        const childFields = c.group.fields;
+        const rows = (await selectRows(em, table, { parent_id: parentId }, { orderBy: { position: 'asc' } }));
+        const acc = [];
+        const metaAcc = [];
+        for (const row of rows) {
+            const { id, parent_id: _pid, position: _pos, ...rest } = row;
+            const { collections: nestedColl, meta: nestedMeta } = await readAtWithMeta(em, table, childFields, id);
+            acc.push({ ...nestEmbeddedAt(childFields, rest), ...nestedColl });
+            metaAcc.push({ id: id, clocks: clockValuesOf(rest), nested: nestedMeta });
+        }
+        collections[c.key] = acc;
+        meta[c.key] = metaAcc;
+    }
+    return { collections, meta };
+}
 export async function readAtMany(em, prefix, fields, parentIds) {
     const out = new Map();
     for (const id of parentIds)
@@ -123,18 +145,17 @@ const modPrefix = (m) => `${m.library}__${m.shelf}`;
 export const splitCollections = (m, input) => splitAt(m.fields, input);
 export const writeCollections = (tx, m, parentId, collections) => writeAt(tx, modPrefix(m), m.fields, parentId, collections);
 export const readCollections = (em, m, parentId) => readAt(em, modPrefix(m), m.fields, parentId);
+export const readCollectionsWithMeta = (em, m, parentId) => readAtWithMeta(em, modPrefix(m), m.fields, parentId);
 export const deleteCollections = (tx, m, parentId) => deleteAt(tx, modPrefix(m), m.fields, parentId);
 function columnsTargets(fields) {
-    return fieldEntries(fields)
-        .filter(([, fm]) => fm.columns)
-        .map(([k, fm]) => [k, Object.keys(fm.columns)]);
+    return fieldEntries(fields).filter(([, fm]) => fm.columns);
 }
 function embeddedGroups(fields) {
     const out = [];
     for (const it of fields) {
-        if (isGroup(it) && isEmbeddedGroup(it))
+        if (hasChildren(it) && isEmbeddedGroup(it))
             out.push([it.key, it]);
-        else if (isGroup(it) && !it.key)
+        else if (hasChildren(it) && it.scope === 'hoist')
             out.push(...embeddedGroups(it.fields));
     }
     return out;
@@ -150,32 +171,22 @@ export function flattenEmbeddedAt(fields, input) {
                 out[`${key}__${sub}`] = val;
         }
     }
-    for (const [key, subs] of columnsTargets(fields)) {
+    for (const [key, fm] of columnsTargets(fields)) {
         if (!(key in out))
             continue;
-        const obj = out[key];
+        const value = out[key];
         delete out[key];
-        for (const s of subs)
-            out[`${key}__${s}`] = obj && typeof obj === 'object' ? obj[s] : null;
+        Object.assign(out, flattenComposite(fm, key, value));
     }
     return out;
 }
 export function nestEmbeddedAt(fields, row) {
     const out = { ...row };
-    for (const [key, subs] of columnsTargets(fields)) {
-        const nested = {};
-        let filled = false;
-        for (const s of subs) {
-            const col = `${key}__${s}`;
-            if (!(col in out))
-                continue;
-            const v = out[col];
-            nested[s] = v;
-            delete out[col];
-            if (v != null && v !== '')
-                filled = true;
-        }
-        if (filled)
+    for (const col of cacheClockColumns(fields))
+        delete out[col];
+    for (const [key, fm] of columnsTargets(fields)) {
+        const nested = nestComposite(fm, key, out, (col) => delete out[col]);
+        if (nested !== undefined)
             out[key] = nested;
     }
     for (const [key, g] of embeddedGroups(fields)) {