@agentshouse/mdruntime 0.1.0-c97456d63baba → 0.1.0-c9b1e59e5baeb

package/dist/read-side.js

@@ -2,6 +2,7 @@ import { and, BasicIndex, configureIndexDevMode, count, createCollection, create
 import { ARRAY_INDEX, parse, pointerSegments } from '@agentshouse/mdmodel';
 import { documentKey } from './rows.js';
 import { after, inUnits, ROW_WEIGHT } from './turn.js';
+import { beyond, cursorOf, MAXIMUM_WINDOW, windowed } from './windows.js';
 function every(expressions) {
     const [first, second, ...rest] = expressions;
     return second === undefined ? first : and(first, second, ...rest);
@@ -59,10 +60,12 @@ function reached(reach, room, document) {
         ? []
         : [or(inArray(room, [...reach.rooms]), inArray(document, [...reach.documents]))];
 }
-function scoped(scope, row, rooms, reach) {
+function scoped(scope, row, rooms, reach, paths = null, folder = null) {
     return every([
         eq(row.kind, 'document'),
         ...(rooms === null ? [] : [inArray(row.room, [...rooms])]),
+        ...(paths === null ? [] : [inArray(row.path, [...paths])]),
+        ...(folder === null ? [] : [eq(row.folder, folder)]),
         ...reached(reach, row.room, row.key),
         ...(scope.types === null ? [] : [inArray(row.type, [...scope.types])]),
         ...(scope.pathPrefix === null ? [] : [inArray(scope.pathPrefix, row.directories)]),
@@ -90,7 +93,7 @@ function related(q, rows, scope, field, kept, reach) {
 }
 function filtered(q, rows, plan) {
     const from = q.from({ d: rows });
-    let query = narrowed(from.where(({ d }) => scoped(plan.root, d, plan.rooms, plan.reach)), plan.root.where, 'd');
+    let query = narrowed(from.where(({ d }) => scoped(plan.root, d, plan.rooms, plan.reach, plan.paths, plan.folder)), plan.root.where, 'd');
     if (plan.hop !== null) {
         const hop = related(q, rows, plan.hop, plan.hop.field, 'source', plan.reach);
         query = query.innerJoin({ h: hop }, ({ d, h }) => eq(d.key, h.key));
@@ -104,25 +107,56 @@ function filtered(q, rows, plan) {
 function bodied(shape) {
     return shape.sections !== null || shape.preamble || shape.markdown;
 }
+function orderingOf(options) {
+    const collated = options.stringSort === 'locale';
+    return {
+        direction: options.direction,
+        nulls: options.nulls ?? 'first',
+        stringSort: options.stringSort ?? 'locale',
+        locale: collated ? options.locale : undefined,
+        localeOptions: collated
+            ? options.localeOptions
+            : undefined,
+    };
+}
+function storedValue(key, row) {
+    return key.of === 'field' ? row.values[key.field] : row[key.of];
+}
+function documentOrder(shape) {
+    return shape.orderBy.map(({ options }) => orderingOf(options));
+}
+function documentCursor(shape, row) {
+    return cursorOf([
+        ...shape.orderBy.map((_, index) => row[`order${index}`]),
+        documentKey(row.room, row.path),
+    ]);
+}
 function documents(q, rows, plan, shape) {
     let query = filtered(q, rows, plan);
+    const cursor = plan.after?.values ?? null;
+    if (cursor !== null) {
+        const orderings = documentOrder(shape);
+        query = query.fn.where(({ d }) => beyond([...shape.orderBy.map(({ key }) => storedValue(key, d)), d.key], cursor, orderings));
+    }
     for (const { key, options } of shape.orderBy) {
         query = query.orderBy(({ d }) => keyed(key, d), options);
     }
-    if (shape.orderBy.length === 0 && plan.limit !== null) {
+    if (shape.orderBy.length > 0 || plan.limit !== null || plan.after !== null) {
         query = query.orderBy(({ d }) => d.key, { stringSort: 'lexical' });
     }
     if (plan.limit !== null)
-        query = query.limit(plan.limit);
+        query = query.limit(plan.limit + 1);
     return query.select(({ d }) => ({
         room: d.room,
         path: d.path,
         type: d.type,
         binary: d.binary,
+        ...Object.fromEntries(shape.orderBy.map(({ key }, index) => [`order${index}`, keyed(key, d)])),
         ...(shape.fields === null ? {} : { fields: d.fields }),
         ...(bodied(shape) ? { text: d.text } : {}),
         ...(shape.backlinks ? { backlinks: toArray(backlinked(q, rows, d, plan.reach)) } : {}),
         ...(shape.change ? { change: d.change } : {}),
+        ...(shape.fileId ? { fileId: d.fileId } : {}),
     }));
 }
 function backlinked(q, rows, document, reach) {
@@ -133,15 +167,13 @@ function backlinked(q, rows, document, reach) {
         .orderBy(({ r }) => r.field, { stringSort: 'lexical' })
         .select(({ r }) => ({ path: r.path, type: r.type, field: r.field }));
 }
-function groups(q, stage, shape, limit) {
+function groups(q, stage, shape) {
     const keys = shape.groupBy.map((_, index) => `key${index}`);
     const from = q.from({ m: stage });
     let query = from.groupBy(({ m }) => keys.map((key) => m[key]));
     for (const { by, direction } of shape.orderBy) {
         query = query.orderBy(({ m }) => (by === 'count' ? count(m.key) : m[keys[by]]), direction);
     }
-    if (limit !== null && shape.orderBy.length > 0)
-        query = query.limit(limit);
     return query.select(({ m }) => ({
         ...Object.fromEntries(keys.map((key) => [key, m[key]])),
         count: count(m.key),
@@ -151,15 +183,27 @@ function built(rows, plan) {
     const shape = plan.shape;
     if (shape.kind === 'documents') {
         const collection = live((q) => documents(q, rows, plan, shape));
-        return { live: collection, dispose: () => void collection.cleanup() };
+        const count = plan.total
+            ? live((q) => filtered(q, rows, plan).select(({ d }) => ({ key: d.key })))
+            : null;
+        return {
+            live: collection,
+            count,
+            dispose: () => {
+                void collection.cleanup();
+                if (count !== null)
+                    void count.cleanup();
+            },
+        };
     }
     const stage = live((q) => filtered(q, rows, plan).select(({ d }) => ({
         key: d.key,
         ...Object.fromEntries(shape.groupBy.map((key, index) => [`key${index}`, keyed(key, d)])),
     })));
-    const collection = live((q) => groups(q, stage, shape, plan.limit));
+    const collection = live((q) => groups(q, stage, shape));
     return {
         live: collection,
+        count: null,
         dispose: () => {
             void collection.cleanup();
             void stage.cleanup();
@@ -239,6 +283,7 @@ function documentOf(key, row, shape, bodies) {
             room: row.room,
             path: row.path,
             ...(binary === null ? {} : { binaryFile: binary }),
+            ...(shape.fileId ? { fileId: row.fileId } : {}),
         };
     }
     const body = bodied(shape) ? bodyOf(row, key, bodies) : null;
@@ -260,33 +305,75 @@ function documentOf(key, row, shape, bodies) {
         ...(shape.markdown && body !== null ? { markdown: body.content } : {}),
         ...(shape.backlinks
             ? {
-                backlinks: row.backlinks.map(({ path, type, field }) => ({ path, type, field })),
+                backlinks: row.backlinks
+                    .slice(0, MAXIMUM_WINDOW)
+                    .map(({ path, type, field }) => ({ path, type, field })),
             }
             : {}),
         ...(shape.change ? { change: row.change } : {}),
+        ...(shape.fileId ? { fileId: row.fileId } : {}),
     };
 }
 function weight(row) {
     return ROW_WEIGHT + (typeof row?.text === 'string' ? row.text.length : 0);
 }
-function rowsOfPlan(plan, entries, bodies, each = (row) => row) {
+const HELD_PER_WEIGHT = 16;
+function groupValues(shape, [key, row]) {
+    return [...shape.orderBy.map(({ by }) => (by === 'count' ? row.count : row[`key${by}`])), key];
+}
+function groupOrder(shape) {
+    return shape.orderBy.map(({ direction }) => ({
+        direction,
+        nulls: 'first',
+        stringSort: 'locale',
+        locale: undefined,
+        localeOptions: undefined,
+    }));
+}
+function windowOf(plan, entries, total) {
     const shape = plan.shape;
+    if (shape.kind === 'documents') {
+        return {
+            ...windowed(entries, plan.limit, plan.after?.text, ([, row]) => documentCursor(shape, row)),
+            total,
+        };
+    }
+    const cursor = plan.after?.values ?? null;
+    const orderings = groupOrder(shape);
+    const following = cursor === null
+        ? entries
+        : entries.filter((entry) => beyond(groupValues(shape, entry), cursor, orderings));
+    return {
+        ...windowed(following, plan.limit, plan.after?.text, (entry) => cursorOf(groupValues(shape, entry))),
+        total: plan.total ? entries.length : undefined,
+    };
+}
+function framed(rows, { next, total }) {
+    return {
+        ...rows,
+        ...(next === undefined ? {} : { next }),
+        ...(total === undefined ? {} : { total }),
+    };
+}
+function rowsOfPlan(plan, entries, total, bodies, each = (row) => row) {
+    const shape = plan.shape;
+    const { kept, ...window } = windowOf(plan, entries, total);
+    const keys = kept.map(([key]) => key);
     if (shape.kind === 'documents') {
         const documents = [];
-        return after(inUnits(entries, ([key, row]) => {
+        return after(inUnits(kept, ([key, row]) => {
             documents.push(each(documentOf(key, row, shape, bodies)));
             return weight(row);
-        }), () => ({ rows: { documents }, keys: entries.map(([key]) => key) }));
+        }), () => ({ rows: framed({ documents }, window), keys, ...window }));
     }
-    const bounded = shape.orderBy.length === 0 && plan.limit !== null ? entries.slice(0, plan.limit) : entries;
     const groups = [];
-    return after(inUnits(bounded, ([, row]) => {
+    return after(inUnits(kept, ([, row]) => {
         groups.push(each({
             key: shape.groupBy.map((_, index) => row[`key${index}`]),
             count: row.count,
         }));
         return ROW_WEIGHT;
-    }), () => ({ rows: { groups }, keys: bounded.map(([key]) => key) }));
+    }), () => ({ rows: framed({ groups }, window), keys, ...window }));
 }
 export function reconciled(previous, next) {
     const before = previous.result;
@@ -311,13 +398,16 @@ export function reconciled(previous, next) {
     const removed = previous.keys.filter((key) => !present.has(key));
     const moved = reordered(previous.keys, next.keys, present);
     const covered = !deepEquals(before.coverage, after.coverage);
-    if (changed.length === 0 && removed.length === 0 && !moved && !covered)
+    const framing = reframed(before, after);
+    if (changed.length === 0 && removed.length === 0 && !moved && !covered && framing === null) {
         return null;
+    }
+    const window = { next: after.next, total: after.total };
     return {
         keyed: {
             result: 'groups' in after
-                ? { groups: rows, coverage: after.coverage }
-                : { documents: rows, coverage: after.coverage },
+                ? framed({ groups: rows, coverage: after.coverage }, window)
+                : framed({ documents: rows, coverage: after.coverage }, window),
             keys: next.keys,
         },
         message: {
@@ -325,9 +415,17 @@ export function reconciled(previous, next) {
             removed,
             order: moved ? next.keys : null,
             ...(covered ? { coverage: after.coverage } : {}),
+            ...framing,
         },
     };
 }
+function reframed(before, after) {
+    const next = before.next === after.next ? {} : { next: after.next ?? null };
+    const total = before.total === after.total || after.total === undefined ? {} : { total: after.total };
+    return Object.keys(next).length === 0 && Object.keys(total).length === 0
+        ? null
+        : { ...next, ...total };
+}
 function reordered(previous, next, present) {
     let at = 0;
     for (const key of previous) {
@@ -357,6 +455,9 @@ export function createReadSide() {
         },
     });
     const shared = new Map();
+    let writing = false;
+    let publishing = null;
+    let closed = false;
     function searched(row) {
         const read = row.text === null ? null : parse(row.text, row.path);
         if (read === null || !read.ok)
@@ -376,8 +477,11 @@ export function createReadSide() {
             return;
         entry.rows.set(key, row);
     }
+    function counted(entry) {
+        return entry.built.count === null ? undefined : entry.built.count.size;
+    }
     function rebuilt(entry) {
-        return after(rowsOfPlan(entry.plan, [...entry.built.live.entries()], entry.bodies), ({ rows: found, keys }) => {
+        return after(rowsOfPlan(entry.plan, [...entry.built.live.entries()], counted(entry), entry.bodies), ({ rows: found, keys, next, total }) => {
             const present = new Set(keys);
             for (const key of [...entry.rows.keys()])
                 if (!present.has(key))
@@ -387,32 +491,34 @@ export function createReadSide() {
                     entry.bodies.delete(key);
             }
             keys.forEach((key, index) => kept(entry, key, rowAt(found, index)));
-            return keys;
+            return { keys, next, total };
         });
     }
     function refreshed(entry, dirty) {
         const shape = entry.plan.shape;
-        const keys = [...entry.built.live.keys()];
-        const named = new Set([...dirty, ...keys.filter((key) => !entry.rows.has(key))]);
-        const values = [...named].map((key) => [key, entry.built.live.get(key)]);
+        const { kept: inWindow, next, total, } = windowOf(entry.plan, [...entry.built.live.entries()], counted(entry));
+        const keys = inWindow.map(([key]) => key);
+        const present = new Set(keys);
+        for (const key of [...entry.rows.keys()]) {
+            if (present.has(key))
+                continue;
+            entry.rows.delete(key);
+            entry.bodies.delete(key);
+        }
+        const values = inWindow.filter(([key]) => dirty.has(key) || !entry.rows.has(key));
         const done = inUnits(values, ([key, value]) => {
-            if (value !== undefined)
-                kept(entry, key, documentOf(key, value, shape, entry.bodies));
-            else {
-                entry.rows.delete(key);
-                entry.bodies.delete(key);
-            }
+            kept(entry, key, documentOf(key, value, shape, entry.bodies));
             return weight(value);
         });
-        return after(done, () => keys);
+        return after(done, () => ({ keys, next, total }));
     }
-    function current(entry, keys, coverage) {
-        const found = keys.map((key) => entry.rows.get(key));
+    function current(entry, found, coverage) {
+        const rows = found.keys.map((key) => entry.rows.get(key));
         return {
             result: entry.plan.shape.kind === 'groups'
-                ? { groups: found, coverage }
-                : { documents: found, coverage },
-            keys,
+                ? framed({ groups: rows, coverage }, found)
+                : framed({ documents: rows, coverage }, found),
+            keys: found.keys,
         };
     }
     function delivered(entry, next) {
@@ -448,25 +554,26 @@ export function createReadSide() {
             const dirty = entry.dirty;
             entry.dirty = new Set();
             entry.whole = false;
-            const keys = whole ? rebuilt(entry) : refreshed(entry, dirty);
-            if (keys instanceof Promise) {
-                void keys
-                    .then((found) => {
+            const found = whole ? rebuilt(entry) : refreshed(entry, dirty);
+            if (found instanceof Promise) {
+                void found
+                    .then((window) => {
                     if (entry.listeners.size > 0)
-                        delivered(entry, current(entry, found, cover.coverage));
-                    run(entry);
+                        delivered(entry, current(entry, window, cover.coverage));
+                    entry.running = false;
+                    wake(entry);
                 })
                     .catch(() => {
                     entry.running = false;
                 });
                 return;
             }
-            delivered(entry, current(entry, keys, cover.coverage));
+            delivered(entry, current(entry, found, cover.coverage));
         }
         entry.running = false;
     }
     function wake(entry) {
-        if (entry.subscription === null || entry.running)
+        if (entry.subscription === null || entry.running || writing || publishing !== null)
             return;
         entry.running = true;
         try {
@@ -492,37 +599,92 @@ export function createReadSide() {
             wake(entry);
         }
     }
+    function committed(next) {
+        if (closed || next.size === 0)
+            return;
+        sync.begin();
+        for (const [key, row] of next) {
+            if (row === null) {
+                if (rows.has(key))
+                    sync.write({ type: 'delete', key });
+                continue;
+            }
+            sync.write({ type: rows.has(key) ? 'update' : 'insert', value: row });
+        }
+        sync.commit();
+    }
+    function published(set) {
+        let unit = new Map();
+        return inUnits(set.documents, (change) => {
+            if (closed)
+                return 0;
+            const shifted = rows.size / HELD_PER_WEIGHT;
+            let spent = 0;
+            for (const row of change.before) {
+                unit.set(row.key, null);
+                spent += ROW_WEIGHT + shifted;
+            }
+            for (const row of change.after) {
+                unit.set(row.key, row);
+                spent += weight(row) + shifted;
+            }
+            return spent;
+        }, () => {
+            committed(unit);
+            unit = new Map();
+        });
+    }
+    function released() {
+        recover();
+        for (const entry of shared.values())
+            wake(entry);
+    }
+    function publishedLater(applied) {
+        const fence = applied.then(() => { }, () => { });
+        publishing = fence;
+        void fence.then(() => {
+            if (publishing !== fence)
+                return;
+            publishing = null;
+            released();
+        });
+        return applied;
+    }
+    async function settled() {
+        while (publishing !== null)
+            await publishing;
+    }
     return {
-        apply(set) {
-            const next = new Map();
-            for (const change of set.documents) {
-                for (const row of change.before)
-                    next.set(row.key, null);
-                for (const row of change.after)
-                    next.set(row.key, row);
+        apply(set, next = () => { }) {
+            const pending = publishing;
+            if (pending !== null) {
+                return publishedLater(pending.then(() => after(published(set), next)));
             }
-            if (next.size > 0) {
-                sync.begin();
-                for (const [key, row] of next) {
-                    if (row === null) {
-                        if (rows.has(key))
-                            sync.write({ type: 'delete', key });
-                        continue;
-                    }
-                    sync.write({ type: rows.has(key) ? 'update' : 'insert', value: row });
-                }
-                sync.commit();
+            writing = true;
+            let applied;
+            try {
+                applied = published(set);
+            }
+            finally {
+                writing = false;
             }
-            recover();
+            if (applied instanceof Promise)
+                return publishedLater(applied.then(next));
+            released();
+            next();
         },
         recover,
+        settled,
         async query(plan, at) {
-            const { live: collection, dispose } = built(rows, plan);
+            const { live: collection, count, dispose } = built(rows, plan);
             try {
                 await collection.toArrayWhenReady();
+                if (count !== null)
+                    await count.toArrayWhenReady();
+                await settled();
                 const entries = [...collection.entries()];
                 const held = at();
-                const found = await rowsOfPlan(plan, entries, null, structuredClone);
+                const found = await rowsOfPlan(plan, entries, count?.size, null, structuredClone);
                 return { rows: found.rows, at: held };
             }
             finally {
@@ -543,13 +705,16 @@ export function createReadSide() {
                     fresh: new Set(),
                     ready: new Set(),
                     subscription: null,
+                    counting: null,
                     whole: true,
                     dirty: new Set(),
                     stale: false,
                     running: false,
                 };
                 created.built.live.on('status:change', ({ status }) => {
-                    if (status === 'cleaned-up' && created.listeners.size === 0)
+                    if (status !== 'cleaned-up' || created.listeners.size > 0)
+                        return;
+                    if (shared.get(identity) === created)
                         shared.delete(identity);
                 });
                 shared.set(identity, created);
@@ -557,7 +722,14 @@ export function createReadSide() {
             }
             const observed = entry;
             observed.listeners.add(deliverTo);
-            observed.built.live.onFirstReady(() => {
+            const ready = (then) => observed.built.live.onFirstReady(() => observed.built.count === null ? then() : observed.built.count.onFirstReady(then));
+            const settled = () => {
+                if (observed.built.count === null)
+                    wake(observed);
+                else
+                    queueMicrotask(() => wake(observed));
+            };
+            ready(() => {
                 if (!observed.listeners.has(deliverTo))
                     return;
                 observed.fresh.add(deliverTo);
@@ -569,8 +741,13 @@ export function createReadSide() {
                         else
                             for (const { key } of changes)
                                 observed.dirty.add(key);
-                        wake(observed);
+                        settled();
                     }, { includeInitialState: false });
+                    observed.counting =
+                        observed.built.count?.subscribeChanges(() => {
+                            observed.stale = true;
+                            settled();
+                        }, { includeInitialState: false }) ?? null;
                 }
                 wake(observed);
             });
@@ -581,11 +758,16 @@ export function createReadSide() {
                     return;
                 observed.subscription?.unsubscribe();
                 observed.subscription = null;
+                observed.counting?.unsubscribe();
+                observed.counting = null;
                 observed.result = undefined;
                 observed.rows.clear();
                 observed.bodies.clear();
                 observed.dirty.clear();
                 observed.whole = true;
+                if (shared.get(identity) === observed)
+                    shared.delete(identity);
+                observed.built.dispose();
             };
         },
         room(room) {
@@ -596,11 +778,13 @@ export function createReadSide() {
             return row === undefined || row.type === null ? null : searched(row);
         },
         close() {
+            closed = true;
             for (const entry of shared.values()) {
                 entry.listeners.clear();
                 entry.fresh.clear();
                 entry.ready.clear();
                 entry.subscription?.unsubscribe();
+                entry.counting?.unsubscribe();
                 entry.built.dispose();
             }
             shared.clear();
@@ -625,13 +809,16 @@ function changedFrom(previous, next) {
     const removed = previous.keys.filter((key) => !present.has(key));
     const moved = reordered(previous.keys, next.keys, present);
     const covered = !deepEquals(before.coverage, after.coverage);
-    if (changed.length === 0 && removed.length === 0 && !moved && !covered)
+    const framing = reframed(before, after);
+    if (changed.length === 0 && removed.length === 0 && !moved && !covered && framing === null) {
         return null;
+    }
     return {
         changed,
         removed,
         order: moved ? next.keys : null,
         ...(covered ? { coverage: after.coverage } : {}),
+        ...framing,
     };
 }
 function rowAt(rows, index) {

package/dist/reads.d.ts

@@ -0,0 +1,11 @@
+import type { Adapter } from './adapter.js';
+import type { Reads } from './bootstrap.js';
+export type DoorReads = Reads & {
+    carried(locator: string): number;
+    finished(locator: string, turn: number): void;
+    opened(): void;
+    applied(): void;
+    lost(): void;
+};
+export declare function doorReads(adapter: Adapter, declaration: (locator: string) => unknown, generation: () => number): DoorReads;
+//# sourceMappingURL=reads.d.ts.map

package/dist/reads.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reads.d.ts","sourceRoot":"","sources":["../src/reads.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EAOR,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAE5C,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG;IAC9B,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACjC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,MAAM,IAAI,IAAI,CAAC;IACf,OAAO,IAAI,IAAI,CAAC;IAChB,IAAI,IAAI,IAAI,CAAC;CACd,CAAC;AAiCF,wBAAgB,SAAS,CACvB,OAAO,EAAE,OAAO,EAChB,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,EACzC,UAAU,EAAE,MAAM,MAAM,GACvB,SAAS,CA0FX"}