@agentshouse/mdruntime-families 0.1.0-cdecd9be0238f → 0.1.0-ce42360f45634

package/README.md

@@ -40,12 +40,15 @@ compare for equality, a list for membership, and the suffixes `_gt`, `_gte`,
 `lt`, `gt`, `lte`, `gte`, `in`, `nin`, `or` and `and`. A field the Room schema
 declares a reference, at any depth, holds the referenced record's id; `id` and
 a reference compare only for equality and membership. Pages and totals are
-counted over every matching record. react-admin's `update` writes the fields
+counted over every matching record, and a page reads only its own window of
+the Room; a list asked for without a page reads every record, window by
+window. react-admin's `update` writes the fields
 that differ from `previousData`, based on it, so the Save of a field another
 writer changed meanwhile is refused; Refine's `update` and either `updateMany`
 compare with the stored record. A list stays current: react-admin's queries of
-a resource are invalidated, and Refine's live mode refetches, when a document
-of the resource changes.
+a resource are invalidated, and Refine's live mode refetches, when a record a
+page or a read shows changes, a record a filter names starts or stops matching
+it, or a page's total changes.
 
 ## License
 

package/dist/react-admin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"react-admin.d.ts","sourceRoot":"","sources":["../src/react-admin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,KAAK,EAEV,YAAY,EAYb,MAAM,SAAS,CAAC;AACjB,OAAO,EAAiD,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAE7F,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAExD,MAAM,MAAM,YAAY,GAAG;IACzB,iBAAiB,CAAC,OAAO,EAAE;QAAE,QAAQ,EAAE,SAAS,OAAO,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAChF,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,YAAY,CAAC;CACpC,CAAC;AAiCF,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,GAAG,YAAY,CAoFhG"}
+{"version":3,"file":"react-admin.d.ts","sourceRoot":"","sources":["../src/react-admin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,KAAK,EAEV,YAAY,EAYb,MAAM,SAAS,CAAC;AACjB,OAAO,EAAgD,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAE5F,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAExD,MAAM,MAAM,YAAY,GAAG;IACzB,iBAAiB,CAAC,OAAO,EAAE;QAAE,QAAQ,EAAE,SAAS,OAAO,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAChF,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,YAAY,CAAC;CACpC,CAAC;AAgCF,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,GAAG,YAAY,CAqFhG"}

package/dist/react-admin.js

@@ -17,11 +17,10 @@ function clause(key, value) {
         value,
     };
 }
-function paged(entries, pagination) {
+function pageOf(pagination) {
     if (pagination === undefined)
-        return [...entries];
-    const start = (pagination.page - 1) * pagination.perPage;
-    return entries.slice(start, start + pagination.perPage);
+        return null;
+    return { start: (pagination.page - 1) * pagination.perPage, size: pagination.perPage };
 }
 export function dataProvider(client, options) {
     const held = records(client, options.resources);
@@ -40,10 +39,10 @@ export function dataProvider(client, options) {
     async function listed(resource, { filter, sort, pagination }) {
         await watched(resource);
         const named = Object.entries(filter ?? {});
-        const entries = await held.list(resource, named.length === 0 ? null : { every: named.map(([key, value]) => clause(key, value)) }, sort === undefined
+        const { entries, total } = await held.list(resource, named.length === 0 ? null : { every: named.map(([key, value]) => clause(key, value)) }, sort === undefined
             ? []
-            : [{ field: sort.field, direction: sort.order === 'DESC' ? 'desc' : 'asc' }]);
-        return { data: paged(entries, pagination), total: entries.length };
+            : [{ field: sort.field, direction: sort.order === 'DESC' ? 'desc' : 'asc' }], pageOf(pagination));
+        return { data: [...entries], total };
     }
     return {
         getList: async (resource, params) => (await listed(resource, params)),

package/dist/records.d.ts

@@ -24,8 +24,16 @@ export type Clause = {
     readonly operator: Operator;
     readonly value: unknown;
 };
+export type Page = {
+    readonly start: number;
+    readonly size: number;
+};
+export type Listed = {
+    readonly entries: readonly Entry[];
+    readonly total: number;
+};
 export declare function records(client: MdRuntimeClient, resources: Resources): {
-    list(name: string, clause: Clause | null, order: readonly Order[]): Promise<Entry[]>;
+    list(name: string, clause: Clause | null, order: readonly Order[], page: Page | null): Promise<Listed>;
     many(name: string, ids: readonly Key[]): Promise<Entry[]>;
     one(name: string, id: Key): Promise<Entry>;
     create(name: string, data: Fields): Promise<Entry>;

package/dist/records.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"records.d.ts","sourceRoot":"","sources":["../src/records.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAGrE,MAAM,MAAM,QAAQ,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1E,MAAM,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE3D,MAAM,MAAM,GAAG,GAAG,MAAM,GAAG,MAAM,CAAC;AAElC,MAAM,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEvD,MAAM,MAAM,KAAK,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAE/E,MAAM,MAAM,KAAK,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,MAAM,CAAA;CAAE,CAAC;AAEnF,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;AAExF,MAAM,MAAM,MAAM,GACd;IAAE,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACrC;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACpC;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AA8JrF,wBAAgB,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,SAAS;IAkJ3D,IAAI,OAAO,MAAM,UAAU,MAAM,GAAG,IAAI,SAAS,SAAS,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAcpF,IAAI,OAAO,MAAM,OAAO,SAAS,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAOzD,GAAG,OAAO,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC;IAM1C,MAAM,OAAO,MAAM,QAAQ,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;IAiBlD,MAAM,OACJ,MAAM,OACP,SAAS,GAAG,EAAE,QACb,MAAM,SACL,MAAM,GAAG,IAAI,GACnB,OAAO,CAAC,KAAK,EAAE,CAAC;IAab,MAAM,OAAO,MAAM,OAAO,SAAS,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAiB3D,KAAK,OAAO,MAAM,SAAS,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC;EAWpE"}
+{"version":3,"file":"records.d.ts","sourceRoot":"","sources":["../src/records.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAErE,MAAM,MAAM,QAAQ,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1E,MAAM,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE3D,MAAM,MAAM,GAAG,GAAG,MAAM,GAAG,MAAM,CAAC;AAElC,MAAM,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEvD,MAAM,MAAM,KAAK,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAE/E,MAAM,MAAM,KAAK,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,MAAM,CAAA;CAAE,CAAC;AAEnF,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;AAExF,MAAM,MAAM,MAAM,GACd;IAAE,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACrC;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACpC;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAErF,MAAM,MAAM,IAAI,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAErE,MAAM,MAAM,MAAM,GAAG;IAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,KAAK,EAAE,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAiTpF,wBAAgB,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,SAAS;IAob3D,IAAI,OACF,MAAM,UACJ,MAAM,GAAG,IAAI,SACd,SAAS,KAAK,EAAE,QACjB,IAAI,GAAG,IAAI,GAChB,OAAO,CAAC,MAAM,CAAC;IAuBZ,IAAI,OAAO,MAAM,OAAO,SAAS,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAOzD,GAAG,OAAO,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC;IAM1C,MAAM,OAAO,MAAM,QAAQ,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;IAqBlD,MAAM,OACJ,MAAM,OACP,SAAS,GAAG,EAAE,QACb,MAAM,SACL,MAAM,GAAG,IAAI,GACnB,OAAO,CAAC,KAAK,EAAE,CAAC;IAab,MAAM,OAAO,MAAM,OAAO,SAAS,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAkB3D,KAAK,OAAO,MAAM,SAAS,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC;EAmBpE"}

package/dist/records.js

@@ -1,9 +1,11 @@
 import { ARRAY_INDEX, equalValue, escapePointer, itemFacts, memberFacts, pointerSegments, references, relative, target, } from '@agentshouse/mdmodel';
-import { saveRoomDrafts } from '@agentshouse/mdruntime/react';
 const FIELDS = { fields: 'all' };
 const REMOVABLE = { fields: 'all', markdown: true };
 const BY_ID = [{ field: 'id', direction: 'asc' }];
 const RESERVED = new Set(['id', 'type']);
+const TRUE = { kind: 'true' };
+const FALSE = { kind: 'false' };
+const WATCHED = new WeakMap();
 const EQUALITY = {
     $eq: true,
     $in: true,
@@ -124,7 +126,128 @@ function byRoom(items) {
     }
     return rooms;
 }
+function complete(answer) {
+    return !('pending' in answer);
+}
+function placeKey({ room, path }) {
+    return JSON.stringify([room, path]);
+}
+function joined(kind, items) {
+    const absorbing = kind === 'and' ? FALSE : TRUE;
+    const neutral = kind === 'and' ? TRUE : FALSE;
+    const kept = items
+        .flatMap((item) => (item.kind === kind ? item.items : [item]))
+        .filter((item) => item.kind !== neutral.kind);
+    if (kept.some((item) => item.kind === absorbing.kind))
+        return absorbing;
+    if (kept.length === 0)
+        return neutral;
+    return kept.length === 1 ? kept[0] : { kind, items: kept };
+}
+function formulaOf(shelf, room, clause) {
+    if ('every' in clause) {
+        return joined('and', clause.every.map((one) => formulaOf(shelf, room, one)));
+    }
+    if ('some' in clause) {
+        return joined('or', clause.some.map((one) => formulaOf(shelf, room, one)));
+    }
+    const { field, operator, value } = clause;
+    const reference = [...shelf.definitions.values()].some((definition) => referring(factsAt(definition.frontmatter, field)));
+    if (field !== 'id' && !reference) {
+        return { kind: 'where', where: { [pointer(field)]: { [operator]: value } } };
+    }
+    const equal = equality(field, operator);
+    const places = listed(value).map((id) => (id === null ? null : placeOf(shelf, id)));
+    const paths = places.flatMap((place) => place !== null && place.room === room ? [place.path] : []);
+    if (field === 'id') {
+        if (paths.length === 0)
+            return equal ? FALSE : TRUE;
+        return { kind: 'path', paths: new Set(paths), holds: equal };
+    }
+    const values = [...paths, ...(places.includes(null) ? [null] : [])];
+    if (values.length === 0 || !referring(factsAt(shelf.definitions.get(room)?.frontmatter, field))) {
+        return equal ? FALSE : TRUE;
+    }
+    return { kind: 'where', where: { [pointer(field)]: { [equal ? '$in' : '$nin']: values } } };
+}
+function namedPaths(formula) {
+    if (formula.kind === 'path')
+        return [...formula.paths];
+    if (formula.kind === 'and' || formula.kind === 'or') {
+        return [...new Set(formula.items.flatMap(namedPaths))];
+    }
+    return [];
+}
+function decided(formula, path) {
+    if (formula.kind === 'path') {
+        return (path !== null && formula.paths.has(path)) === formula.holds ? TRUE : FALSE;
+    }
+    if (formula.kind === 'and' || formula.kind === 'or') {
+        return joined(formula.kind, formula.items.map((item) => decided(item, path)));
+    }
+    return formula;
+}
+function whereOf(formula) {
+    if (formula.kind === 'where')
+        return formula.where;
+    if (formula.kind === 'and')
+        return { $and: formula.items.map((item) => whereOf(item) ?? {}) };
+    if (formula.kind === 'or')
+        return { $or: formula.items.map((item) => whereOf(item) ?? {}) };
+    return undefined;
+}
+function filtered(where) {
+    return where === undefined ? {} : { where };
+}
+function afterOf(after) {
+    return after === undefined ? {} : { after };
+}
+function watchedOf(client) {
+    const held = WATCHED.get(client);
+    if (held !== undefined)
+        return held;
+    const fresh = {
+        walks: new Map(),
+        listeners: new Map(),
+        read: new Map(),
+        watching: new Map(),
+    };
+    WATCHED.set(client, fresh);
+    return fresh;
+}
 export function records(client, resources) {
+    const { walks, listeners, read, watching } = watchedOf(client);
+    async function answerOf(query) {
+        const answer = await client.query(query);
+        if (complete(answer))
+            return answer;
+        const observation = await client.observe('query', query);
+        try {
+            return await new Promise((resolve) => {
+                const settle = (value) => {
+                    if (complete(value))
+                        resolve(value);
+                };
+                observation.subscribe(settle);
+                settle(observation.current());
+            });
+        }
+        finally {
+            observation.close();
+        }
+    }
+    async function everyOf(query) {
+        const walked = { documents: [], windows: [] };
+        let after;
+        do {
+            const window = { ...query, ...afterOf(after) };
+            const answer = await answerOf(window);
+            walked.windows.push(window);
+            walked.documents.push(...answer.documents);
+            after = answer.next;
+        } while (after !== undefined);
+        return walked;
+    }
     function resourceOf(name) {
         const resource = Object.hasOwn(resources, name) ? resources[name] : undefined;
         if (resource === undefined)
@@ -137,7 +260,7 @@ export function records(client, resources) {
         const definitions = new Map();
         for (const room of rooms) {
             const schema = (await client.types(room)).collection;
-            if (schema.status === 'unavailable')
+            if (schema.types === null)
                 continue;
             const definition = schema.types.find((type) => type.name === resource.type);
             if (definition === undefined)
@@ -146,98 +269,308 @@ export function records(client, resources) {
         }
         return { resource, rooms, definitions };
     }
-    async function keys(shelf, query) {
-        const { documents } = await client.query({
-            ...query,
-            types: [shelf.resource.type],
-            pathPrefix: shelf.resource.folder,
-            select: {},
+    function folderOf(shelf) {
+        return { types: [shelf.resource.type], pathPrefix: shelf.resource.folder };
+    }
+    function closed(observations) {
+        void observations?.then((all) => {
+            for (const observation of all)
+                observation.close();
+        }, () => { });
+    }
+    function touched(name) {
+        walks.delete(name);
+        read.delete(name);
+        const held = watching.get(name);
+        if (held !== undefined) {
+            watching.set(name, new Map());
+            for (const observations of held.values())
+                closed(observations);
+        }
+        for (const heard of listeners.get(name) ?? [])
+            heard();
+    }
+    function observed(name, view, queries) {
+        const held = watching.get(name);
+        if (held === undefined)
+            return;
+        const before = held.get(view);
+        const opening = Promise.all(queries.map(async (query) => {
+            const observation = await client.observe('query', query);
+            observation.subscribe(() => {
+                if (watching.get(name)?.get(view) === opening)
+                    touched(name);
+            });
+            return observation;
+        }));
+        held.set(view, opening);
+        opening.catch(() => {
+            if (watching.get(name)?.get(view) === opening)
+                held.delete(view);
         });
-        return new Set(documents.map((document) => keyOf(shelf, document)));
+        closed(before);
     }
-    async function referenced(shelf, field, places) {
-        const kept = new Set();
+    function remember(name, view, queries) {
+        const views = read.get(name) ?? new Map();
+        views.set(view, queries);
+        read.set(name, views);
+        observed(name, view, queries);
+    }
+    async function selectionOf(shelf, clause) {
+        const streams = new Map();
+        const named = [];
+        const lookups = [];
         for (const room of shelf.rooms) {
-            const paths = places.flatMap((place) => place === null ? [null] : place.room === room ? [place.path] : []);
-            const definition = shelf.definitions.get(room);
-            if (paths.length === 0 || !referring(factsAt(definition?.frontmatter, field)))
-                continue;
-            const where = { [pointer(field)]: { $in: paths } };
-            for (const key of await keys(shelf, { rooms: [room], where }))
-                kept.add(key);
-        }
-        return kept;
-    }
-    async function matching(shelf, all, clause) {
-        if ('every' in clause) {
-            let kept = new Set(all.map((document) => keyOf(shelf, document)));
-            for (const one of clause.every) {
-                const next = await matching(shelf, all, one);
-                kept = new Set([...kept].filter((key) => next.has(key)));
+            const formula = clause === null ? TRUE : formulaOf(shelf, room, clause);
+            const paths = namedPaths(formula);
+            const rest = decided(formula, null);
+            if (rest.kind !== 'false') {
+                const where = whereOf(rest);
+                const key = JSON.stringify(where ?? null);
+                const stream = streams.get(key) ?? { rooms: [], where, excluded: new Set() };
+                stream.rooms.push(room);
+                for (const path of paths)
+                    stream.excluded.add(placeKey({ room, path }));
+                streams.set(key, stream);
+            }
+            const each = new Map();
+            for (const path of paths) {
+                const kept = decided(formula, path);
+                if (kept.kind === 'false')
+                    continue;
+                const where = whereOf(kept);
+                const key = JSON.stringify(where ?? null);
+                const held = each.get(key) ?? { where, paths: [] };
+                held.paths.push(path);
+                each.set(key, held);
             }
-            return kept;
+            for (const { where, paths: these } of each.values()) {
+                const lookup = {
+                    ...folderOf(shelf),
+                    rooms: [room],
+                    paths: these,
+                    ...filtered(where),
+                    select: {},
+                };
+                const found = await everyOf(lookup);
+                lookups.push(...found.windows);
+                named.push(...found.documents.map(({ room: at, path }) => ({ room: at, path })));
+            }
+        }
+        return { streams: [...streams.values()], named, lookups };
+    }
+    function streamQuery(shelf, stream, orderBy) {
+        return {
+            ...folderOf(shelf),
+            rooms: stream.rooms,
+            ...filtered(stream.where),
+            orderBy,
+            select: {},
+        };
+    }
+    function walkOf(name, query) {
+        const fresh = { total: null, cursors: new Map([[0, undefined]]) };
+        if (!watching.has(name))
+            return fresh;
+        const walked = walks.get(name) ?? new Map();
+        walks.set(name, walked);
+        const key = JSON.stringify(query);
+        const walk = walked.get(key) ?? fresh;
+        walked.set(key, walk);
+        return walk;
+    }
+    async function cursorAt({ cursors }, query, start) {
+        let at = Math.max(...[...cursors.keys()].filter((one) => one <= start));
+        let after = cursors.get(at);
+        while (at < start) {
+            const window = await answerOf({ ...query, limit: start - at, ...afterOf(after) });
+            if (window.next === undefined)
+                return null;
+            at += window.documents.length;
+            after = window.next;
+            cursors.set(at, after);
         }
-        if ('some' in clause) {
-            const kept = new Set();
-            for (const one of clause.some) {
-                for (const key of await matching(shelf, all, one))
-                    kept.add(key);
+        return after;
+    }
+    function prefixOf({ cursors }, query, start) {
+        const at = [...cursors.keys()]
+            .filter((one) => one <= start)
+            .sort((left, right) => left - right);
+        return at.slice(1).map((to, index) => ({
+            ...query,
+            limit: to - at[index],
+            ...afterOf(cursors.get(at[index])),
+        }));
+    }
+    async function paged(name, shelf, view, query, { start, size }, again = true) {
+        const walk = walkOf(name, query);
+        const after = await cursorAt(walk, query, start);
+        const window = {
+            ...query,
+            ...(after === null ? { limit: 0 } : { select: FIELDS, limit: size, ...afterOf(after) }),
+            total: true,
+        };
+        const page = await answerOf(window);
+        const total = page.total ?? 0;
+        if (walk.total !== null && walk.total !== total && again) {
+            walks.get(name)?.delete(JSON.stringify(query));
+            return paged(name, shelf, view, query, { start, size }, false);
+        }
+        walk.total = total;
+        if (page.next !== undefined && after !== null) {
+            walk.cursors.set(start + page.documents.length, page.next);
+        }
+        remember(name, view, [
+            ...(start === 0 || after === null ? [] : prefixOf(walk, query, start)),
+            after === null ? { ...query, limit: start, total: true } : window,
+        ]);
+        return { entries: page.documents.map((document) => entryOf(shelf, document)), total };
+    }
+    async function sortedPlaces(shelf, places, orderBy) {
+        if (places.length <= 1)
+            return [...places];
+        const wanted = new Set(places.map(placeKey));
+        const sorted = await everyOf({
+            ...folderOf(shelf),
+            rooms: [...new Set(places.map(({ room }) => room))],
+            paths: [...new Set(places.map(({ path }) => path))],
+            orderBy,
+            select: {},
+        });
+        return sorted.documents.flatMap(({ room, path }) => wanted.has(placeKey({ room, path })) ? [{ room, path }] : []);
+    }
+    function streamSource(query, excluded) {
+        let after;
+        const source = {
+            buffer: [],
+            more: true,
+            windows: [],
+            async read(limit) {
+                const asked = { ...query, ...(limit === null ? {} : { limit }), ...afterOf(after) };
+                const window = await answerOf(asked);
+                after = window.next;
+                source.more = after !== undefined;
+                source.windows.push(asked);
+                source.buffer.push(...window.documents.flatMap(({ room, path }) => excluded.has(placeKey({ room, path })) ? [] : [{ room, path }]));
+            },
+        };
+        return source;
+    }
+    async function mergedPlaces(shelf, selection, orderBy, end) {
+        const queries = selection.streams.map((stream) => streamQuery(shelf, stream, orderBy));
+        const streams = selection.streams.map((stream, at) => streamSource(queries[at], stream.excluded));
+        const sources = [
+            ...streams,
+            {
+                buffer: await sortedPlaces(shelf, selection.named, orderBy),
+                more: false,
+                windows: [],
+                read: async () => { },
+            },
+        ];
+        const merged = [];
+        while (merged.length < end) {
+            const wanted = end - merged.length;
+            for (const source of sources) {
+                while (source.buffer.length === 0 && source.more) {
+                    await source.read(Number.isFinite(wanted) ? wanted : null);
+                }
             }
-            return kept;
+            const filled = sources.filter((source) => source.buffer.length > 0);
+            if (filled.length === 0)
+                break;
+            const order = filled.length === 1
+                ? [...filled[0].buffer]
+                : await sortedPlaces(shelf, filled.flatMap((source) => source.buffer), orderBy);
+            const at = order.map(placeKey);
+            const cut = Math.min(order.length - 1, ...filled
+                .filter((source) => source.more)
+                .map((source) => at.indexOf(placeKey(source.buffer[source.buffer.length - 1]))));
+            const emitted = order.slice(0, cut + 1);
+            const gone = new Set(emitted.map(placeKey));
+            for (const source of filled) {
+                source.buffer = source.buffer.filter((place) => !gone.has(placeKey(place)));
+            }
+            merged.push(...emitted);
         }
-        const { field, operator, value } = clause;
-        const reference = [...shelf.definitions.values()].some((definition) => referring(factsAt(definition.frontmatter, field)));
-        if (field !== 'id' && !reference) {
-            return keys(shelf, { where: { [pointer(field)]: { [operator]: value } } });
+        return {
+            places: merged.slice(0, end),
+            windows: streams.flatMap((source, at) => [
+                { ...queries[at], limit: 0, total: true },
+                ...source.windows,
+            ]),
+        };
+    }
+    async function totalOf(shelf, selection, orderBy) {
+        let total = selection.named.length;
+        for (const stream of selection.streams) {
+            const query = streamQuery(shelf, stream, orderBy);
+            total += (await answerOf({ ...query, limit: 0, total: true })).total ?? 0;
+            const excluded = new Map();
+            for (const key of stream.excluded) {
+                const [room, path] = JSON.parse(key);
+                excluded.set(room, [...(excluded.get(room) ?? []), path]);
+            }
+            for (const [room, paths] of excluded) {
+                const counted = { ...query, rooms: [room], paths, limit: 0, total: true };
+                total -= (await answerOf(counted)).total ?? 0;
+            }
         }
-        const equal = equality(field, operator);
-        const places = listed(value).map((id) => (id === null ? null : placeOf(shelf, id)));
-        const matched = field === 'id'
-            ? new Set(places.flatMap((place) => place === null ? [] : [idOf(shelf, place.room, place.path)]))
-            : await referenced(shelf, field, places);
-        return new Set(all.map((document) => keyOf(shelf, document)).filter((key) => matched.has(key) === equal));
-    }
-    async function found(shelf, ids, select) {
-        const places = ids.map((id) => placeOf(shelf, id));
+        return total;
+    }
+    async function placed(shelf, places, select) {
+        const queries = [];
         const documents = new Map();
-        for (const room of new Set(places.map((place) => place.room))) {
-            const answer = await client.query({
+        const rooms = new Map();
+        for (const { room, path } of places) {
+            rooms.set(room, (rooms.get(room) ?? new Set()).add(path));
+        }
+        for (const [room, paths] of rooms) {
+            const walked = await everyOf({
+                ...folderOf(shelf),
                 rooms: [room],
-                types: [shelf.resource.type],
-                pathPrefix: shelf.resource.folder,
+                paths: [...paths],
                 select,
             });
-            for (const document of answer.documents)
-                documents.set(keyOf(shelf, document), document);
+            queries.push(...walked.windows);
+            for (const document of walked.documents)
+                documents.set(placeKey(document), document);
         }
-        return places.map((place) => ({
-            place,
-            document: documents.get(idOf(shelf, place.room, place.path)) ?? null,
-        }));
+        return { queries, documents };
+    }
+    async function merged(name, shelf, view, selection, orderBy, page) {
+        const end = page === null ? Number.POSITIVE_INFINITY : page.start + page.size;
+        const { places, windows } = await mergedPlaces(shelf, selection, orderBy, end);
+        const shown = page === null ? places : places.slice(page.start);
+        const { queries, documents } = await placed(shelf, shown, FIELDS);
+        remember(name, view, [...windows, ...selection.lookups, ...queries]);
+        return {
+            entries: shown.flatMap((place) => {
+                const document = documents.get(placeKey(place));
+                return document === undefined ? [] : [entryOf(shelf, document)];
+            }),
+            total: page === null ? places.length : await totalOf(shelf, selection, orderBy),
+        };
     }
-    async function held(shelf, ids, select) {
-        return (await found(shelf, ids, select)).map(({ place, document }) => {
+    async function found(name, shelf, ids, select) {
+        const places = ids.map((id) => placeOf(shelf, id));
+        const { queries, documents } = await placed(shelf, places, select);
+        remember(name, JSON.stringify(['named', places.map(placeKey)]), queries);
+        return places.map((place) => ({ place, document: documents.get(placeKey(place)) ?? null }));
+    }
+    async function held(name, shelf, ids, select) {
+        return (await found(name, shelf, ids, select)).map(({ place, document }) => {
             if (document === null)
                 throw new Error(`no ${shelf.resource.type} is held at ${place.path}`);
             return { place, document };
         });
     }
-    async function dropped(room, paths) {
-        for (const path of paths) {
-            const left = await client.roomDraft.read(room, path);
-            if (left !== null)
-                await client.roomDraft.discard(room, path, left.revision);
-        }
-    }
-    async function saved(room, parts) {
+    async function saved(name, room, parts) {
         if (parts.size === 0)
             return;
-        const result = await saveRoomDrafts(client, room, parts).catch(async (error) => {
-            await dropped(room, parts.keys());
-            throw error;
-        });
+        walks.delete(name);
+        const result = await client.saveParts(room, [...parts].map(([path, one]) => ({ path, parts: one })));
         if (result.outcome === 'refused') {
-            await dropped(room, parts.keys());
             throw Object.assign(new Error(`the Save was refused as ${result.code}`, { cause: result }), {
                 code: result.code,
                 retryable: result.retryable,
@@ -245,26 +578,34 @@ export function records(client, resources) {
         }
     }
     return {
-        async list(name, clause, order) {
+        async list(name, clause, order, page) {
             const shelf = await shelfOf(name);
-            const { documents } = await client.query({
-                types: [shelf.resource.type],
-                pathPrefix: shelf.resource.folder,
-                orderBy: (order.length === 0 ? BY_ID : order).flatMap((one) => sorting(shelf, one)),
-                select: FIELDS,
-            });
-            const kept = clause === null ? null : await matching(shelf, documents, clause);
-            return documents
-                .filter((document) => kept === null || kept.has(keyOf(shelf, document)))
-                .map((document) => entryOf(shelf, document));
+            const selection = await selectionOf(shelf, clause);
+            const orderBy = (order.length === 0 ? BY_ID : order).flatMap((one) => sorting(shelf, one));
+            const view = JSON.stringify(['list', clause, order]);
+            const [stream] = selection.streams;
+            if (selection.streams.length === 1 &&
+                stream.excluded.size === 0 &&
+                selection.named.length === 0) {
+                const query = streamQuery(shelf, stream, orderBy);
+                if (page !== null)
+                    return paged(name, shelf, view, query, page);
+                const all = await everyOf({ ...query, select: FIELDS });
+                remember(name, view, all.windows);
+                return {
+                    entries: all.documents.map((document) => entryOf(shelf, document)),
+                    total: all.documents.length,
+                };
+            }
+            return merged(name, shelf, view, selection, orderBy, page);
         },
         async many(name, ids) {
             const shelf = await shelfOf(name);
-            return (await found(shelf, ids, FIELDS)).flatMap(({ document }) => document === null ? [] : [entryOf(shelf, document)]);
+            return (await found(name, shelf, ids, FIELDS)).flatMap(({ document }) => document === null ? [] : [entryOf(shelf, document)]);
         },
         async one(name, id) {
             const shelf = await shelfOf(name);
-            const [{ document }] = await held(shelf, [id], FIELDS);
+            const [{ document }] = await held(name, shelf, [id], FIELDS);
             return entryOf(shelf, document);
         },
         async create(name, data) {
@@ -278,28 +619,28 @@ export function records(client, resources) {
             const fields = Object.fromEntries(Object.entries(data)
                 .filter(([field, value]) => !RESERVED.has(field) && value !== undefined)
                 .map(([field, value]) => [field, written(shelf, place, field, value)]));
-            await saved(room, new Map([[place.path, { creation: { type, fields, sections: {} } }]]));
-            const [{ document }] = await held(shelf, [idOf(shelf, room, place.path)], FIELDS);
+            await saved(name, room, new Map([[place.path, { creation: { type, fields, sections: {} } }]]));
+            const [{ document }] = await held(name, shelf, [idOf(shelf, room, place.path)], FIELDS);
             return entryOf(shelf, document);
         },
         async update(name, ids, data, shown) {
             const shelf = await shelfOf(name);
-            for (const [room, inRoom] of byRoom(await held(shelf, ids, FIELDS))) {
+            for (const [room, inRoom] of byRoom(await held(name, shelf, ids, FIELDS))) {
                 const parts = new Map();
                 for (const one of inRoom) {
                     const part = changed(shelf, one, data, shown);
                     if (part !== null)
                         parts.set(one.place.path, part);
                 }
-                await saved(room, parts);
+                await saved(name, room, parts);
             }
-            return (await held(shelf, ids, FIELDS)).map(({ document }) => entryOf(shelf, document));
+            return (await held(name, shelf, ids, FIELDS)).map(({ document }) => entryOf(shelf, document));
         },
         async remove(name, ids) {
             const shelf = await shelfOf(name);
-            const removed = await held(shelf, ids, REMOVABLE);
+            const removed = await held(name, shelf, ids, REMOVABLE);
             for (const [room, inRoom] of byRoom(removed)) {
-                await saved(room, new Map(inRoom.map(({ place, document }) => [
+                await saved(name, room, new Map(inRoom.map(({ place, document }) => [
                     place.path,
                     { removal: { base: document.markdown ?? null } },
                 ])));
@@ -307,14 +648,25 @@ export function records(client, resources) {
             return removed.map(({ document }) => entryOf(shelf, document));
         },
         async watch(name, heard) {
-            const { type, folder } = resourceOf(name);
-            const observation = await client.observe('query', {
-                types: [type],
-                pathPrefix: folder,
-                select: { markdown: true },
-            });
-            observation.subscribe(heard);
-            return () => observation.close();
+            resourceOf(name);
+            const hearing = listeners.get(name) ?? new Set();
+            listeners.set(name, hearing);
+            hearing.add(heard);
+            if (!watching.has(name)) {
+                watching.set(name, new Map());
+                for (const [view, queries] of read.get(name) ?? [])
+                    observed(name, view, queries);
+            }
+            return () => {
+                hearing.delete(heard);
+                if (hearing.size > 0)
+                    return;
+                listeners.delete(name);
+                const observations = watching.get(name);
+                watching.delete(name);
+                for (const one of observations?.values() ?? [])
+                    closed(one);
+            };
         },
     };
 }

package/dist/refine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"refine.d.ts","sourceRoot":"","sources":["../src/refine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,KAAK,EAMV,YAAY,EAMZ,YAAY,EAIb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAML,KAAK,SAAS,EACf,MAAM,cAAc,CAAC;AAEtB,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAExD,MAAM,MAAM,eAAe,GAAG;IAAE,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAA;CAAE,CAAC;AAgChE,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,YAAY,CAgF5F;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,YAAY,CAW5F"}
+{"version":3,"file":"refine.d.ts","sourceRoot":"","sources":["../src/refine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,KAAK,EAMV,YAAY,EAMZ,YAAY,EAIb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAML,KAAK,SAAS,EACf,MAAM,cAAc,CAAC;AAEtB,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAExD,MAAM,MAAM,eAAe,GAAG;IAAE,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAA;CAAE,CAAC;AA+BhE,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,YAAY,CAiF5F;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,YAAY,CAW5F"}

package/dist/refine.js

@@ -21,19 +21,18 @@ function clause(filter) {
         throw new Error(`the ${filter.operator} filter has no condition`);
     return { field: filter.field, operator, value: filter.value };
 }
-function paged(entries, pagination) {
+function pageOf(pagination) {
     const { currentPage = 1, pageSize = 10, mode = 'server' } = pagination ?? {};
     if (mode !== 'server')
-        return [...entries];
-    const start = (currentPage - 1) * pageSize;
-    return entries.slice(start, start + pageSize);
+        return null;
+    return { start: (currentPage - 1) * pageSize, size: pageSize };
 }
 export function dataProvider(client, options) {
     const held = records(client, options.resources);
     return {
         getList: async ({ resource, filters, sorters, pagination, }) => {
-            const entries = await held.list(resource, filters === undefined || filters.length === 0 ? null : { every: filters.map(clause) }, (sorters ?? []).map(({ field, order }) => ({ field, direction: order })));
-            return { data: paged(entries, pagination), total: entries.length };
+            const { entries, total } = await held.list(resource, filters === undefined || filters.length === 0 ? null : { every: filters.map(clause) }, (sorters ?? []).map(({ field, order }) => ({ field, direction: order })), pageOf(pagination));
+            return { data: [...entries], total };
         },
         getMany: async ({ resource, ids, }) => ({ data: await held.many(resource, ids) }),
         getOne: async ({ resource, id }) => ({ data: await held.one(resource, id) }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentshouse/mdruntime-families",
-  "version": "0.1.0-cdecd9be0238f",
+  "version": "0.1.0-ce42360f45634",
   "description": "react-admin and Refine data providers over the MD Runtime client",
   "license": "MIT",
   "type": "module",
@@ -28,8 +28,8 @@
     "LICENSE"
   ],
   "dependencies": {
-    "@agentshouse/mdmodel": "0.1.0-c6e675bb19980",
-    "@agentshouse/mdruntime": "0.1.0-cfb2ae0107054"
+    "@agentshouse/mdmodel": "0.1.0-ccf1367190878",
+    "@agentshouse/mdruntime": "0.1.0-c13854b9249b6"
   },
   "peerDependencies": {
     "@refinedev/core": ">=5",