@agentshouse/mdruntime 0.1.0-ca6edc1abd618 → 0.1.0-ca8707258fdde

package/dist/install.js

@@ -0,0 +1,537 @@
+import { attachedModel } from './attached-model.js';
+import { bootstrapRoom } from './bootstrap.js';
+import { refusedForSpace } from './durability.js';
+import { NO_MODEL } from './model.js';
+import { advancement } from './refresh.js';
+import { NO_INSTALL, } from './store.js';
+import { unconfirmed } from './transfer.js';
+import { ROW_WEIGHT } from './turn.js';
+// Bytes still arriving are reported at most this often, so readiness is not rebuilt for every chunk the network hands over while a slow link still moves the figure.
+const ARRIVAL_REPORT_MS = 250;
+const NOTHING_HELD = { bytes: 0 };
+function transferGone(held) {
+    return {
+        ...held,
+        transfer: null,
+        arrived: null,
+        streaming: false,
+        underived: 0,
+        unindexed: [],
+    };
+}
+function amountOf(held, kept) {
+    if (held.transfer !== null) {
+        const owed = held.unindexed.reduce((sum, part) => sum + part.bytes, 0);
+        return {
+            done: held.transfer.done.bytes - Math.ceil(owed),
+            total: Math.max(held.arrived ?? 0, held.transfer.done.bytes),
+        };
+    }
+    return { done: kept, total: held.arrived ?? kept };
+}
+function setBack(held) {
+    const now = amountOf(held, NOTHING_HELD.bytes);
+    const earlier = held.startedOver;
+    return {
+        ...held,
+        startedOver: {
+            done: Math.max(now.done, earlier?.done ?? 0),
+            total: Math.max(now.total, earlier?.total ?? 0),
+        },
+    };
+}
+function owedIndexes(held, bytes, text) {
+    if (bytes <= 0 || text <= 0)
+        return held;
+    return { ...held, unindexed: [...held.unindexed, { bytes, text }] };
+}
+function built(held, done, text) {
+    const owed = [...held.unindexed];
+    let left = text;
+    while (!done && left > 0 && owed.length > 0) {
+        const [first] = owed;
+        if (left >= first.text) {
+            left -= first.text;
+            owed.shift();
+            continue;
+        }
+        owed[0] = {
+            bytes: (first.bytes * (first.text - left)) / first.text,
+            text: first.text - left,
+        };
+        left = 0;
+    }
+    return { ...held, unindexed: done ? [] : owed };
+}
+function batchEnd(entries, from, bytes) {
+    let end = from;
+    let spent = 0;
+    while (end < entries.length) {
+        const size = (entries[end].content?.length ?? 0) + ROW_WEIGHT;
+        if (spent + size > bytes && end > from)
+            break;
+        spent += size;
+        end += 1;
+    }
+    return end;
+}
+export function installationOf(store, coverage, held) {
+    const rooms = { done: 0, total: 0 };
+    const bytes = { done: 0, total: 0 };
+    let installing = 0;
+    let downloading = false;
+    for (const { room, state } of coverage) {
+        if (state === 'unadmitted')
+            continue;
+        rooms.total += 1;
+        if (state === 'complete')
+            rooms.done += 1;
+        const install = store.installs.of(room);
+        const underway = state === 'acquiring' || (install.installing && state !== 'complete');
+        if (underway)
+            installing += 1;
+        if (underway && (install.streaming || install.arrived === null))
+            downloading = true;
+        const kept = underway ? NOTHING_HELD.bytes : (held.get(room)?.bytes ?? NOTHING_HELD.bytes);
+        const now = amountOf(install, kept);
+        const earlier = underway ? install.startedOver : null;
+        bytes.done += Math.max(now.done, earlier?.done ?? 0);
+        bytes.total += Math.max(now.total, earlier?.total ?? 0);
+    }
+    const step = installing === 0
+        ? null
+        : downloading
+            ? 'downloading'
+            : bytes.done < bytes.total
+                ? 'writing'
+                : 'replaying';
+    return { step, rooms, bytes };
+}
+export function roomInFlight(store, locator) {
+    const { catching, transfer } = store.installs.of(locator);
+    return catching || transfer !== null;
+}
+function installing(installer, locator) {
+    installer.store.installs.change(locator, (held) => ({ ...held, installing: true }));
+    installer.queueReadiness();
+}
+function installed(installer, locator) {
+    if (!installer.store.installs.of(locator).installing)
+        return;
+    installer.store.installs.change(locator, (held) => ({ ...held, installing: false }));
+    installer.queueReadiness();
+}
+export function forgetInstall(store, locator) {
+    store.installs.change(locator, ({ installing }) => ({ ...NO_INSTALL, installing }));
+}
+export function caughtUpTo(installer, locator) {
+    if (!installer.store.installs.of(locator).catching)
+        return;
+    installer.store.installs.change(locator, (held) => ({ ...held, catching: false }));
+    installer.recover();
+}
+export function catching(store, locator) {
+    store.installs.change(locator, (held) => ({ ...held, catching: true }));
+}
+export function confirmedAgain(store, locator) {
+    store.installs.change(locator, (held) => ({
+        ...held,
+        confirmations: held.confirmations + 1,
+    }));
+}
+export function standingAt(store, locator, held) {
+    return { key: held?.key ?? null, confirmed: store.installs.of(locator).confirmations };
+}
+function standsAt(store, locator, held, from) {
+    const now = standingAt(store, locator, held);
+    return now.key === from.key && now.confirmed === from.confirmed;
+}
+function shownAt({ store, socket }, locator, held) {
+    const raised = store.installs.of(locator).highest ?? undefined;
+    if (held === undefined || raised === undefined)
+        return held ?? raised;
+    return socket.reads.reached(locator, held, raised) ? held : raised;
+}
+export function showing(system, locator, roomVersion, held) {
+    const at = shownAt(system, locator, held);
+    if (at === undefined || !system.socket.reads.reached(locator, at, roomVersion)) {
+        system.store.installs.change(locator, (room) => ({ ...room, highest: roomVersion }));
+    }
+}
+export function olderThan(system, locator, roomVersion, held) {
+    const { store, socket } = system;
+    if (store.installs.of(locator).beyondHorizon)
+        return false;
+    const at = shownAt(system, locator, held);
+    return (at !== undefined &&
+        socket.reads.reached(locator, at, roomVersion) &&
+        !socket.reads.reached(locator, roomVersion, at));
+}
+async function older(installer, locator, roomVersion) {
+    const { store } = installer;
+    if (store.installs.of(locator).beyondHorizon)
+        return false;
+    return olderThan(installer, locator, roomVersion, (await store.record(locator))?.roomVersion);
+}
+export function replaced(store, locators) {
+    for (const locator of locators) {
+        store.installs.change(locator, (held) => ({ ...held, highest: null, beyondHorizon: true }));
+    }
+}
+export async function reachedBy({ store, socket }, locator, roomVersion) {
+    const held = await store.record(locator);
+    return held !== undefined && socket.reads.reached(locator, held.roomVersion, roomVersion);
+}
+export async function behind({ store, socket }, room) {
+    const held = (await store.record(room.locator))?.roomVersion;
+    return held !== undefined && !socket.reads.reached(room.locator, held, room.roomVersion);
+}
+export async function writtenAfter({ store, socket }, locator, roomVersion, path) {
+    const held = await store.record(locator);
+    if (held === undefined)
+        return false;
+    const [entry] = await (await store.open()).entries(held.key, 'confirmed', [path]);
+    const written = entry?.written ?? undefined;
+    return (written !== undefined &&
+        socket.reads.reached(locator, written, roomVersion) &&
+        !socket.reads.reached(locator, roomVersion, written));
+}
+async function writtenLater(installer, local, locator, key, paths, roomVersion, heldAt) {
+    if (paths.length === 0)
+        return new Set();
+    const { reached } = installer.socket.reads;
+    const behindShown = heldAt === undefined
+        ? await older(installer, locator, roomVersion)
+        : olderThan(installer, locator, roomVersion, heldAt);
+    if (!behindShown)
+        return new Set();
+    const entries = await local.entries(key, 'confirmed', paths);
+    return new Set(entries.flatMap(({ path, written }) => written !== null &&
+        reached(locator, written, roomVersion) &&
+        !reached(locator, roomVersion, written)
+        ? [path]
+        : []));
+}
+function remodel(installer, answers, held) {
+    const answer = answers?.collection;
+    if (answer === undefined)
+        return held;
+    return { collection: attachedModel(answer, installer.roomTypes) };
+}
+function refreshingOf(installer, local, locator, key, composition, liveness, replaying = false, heldAt) {
+    const { store } = installer;
+    const write = replaying ? store.finishingWrite : store.exclusive;
+    return {
+        turn() {
+            if (replaying)
+                installing(installer, locator);
+        },
+        prior: async (paths) => new Map((await local.entries(key, 'refreshed', paths)).map((entry) => [entry.path, entry])),
+        later: (paths, roomVersion) => writtenLater(installer, local, locator, key, paths, roomVersion, heldAt),
+        async stage(page) {
+            for (let written = 0;;) {
+                const end = batchEnd(page.entries, written, store.stageBytes());
+                const last = end === page.entries.length;
+                const entries = page.entries.slice(written, end);
+                const staged = await write(async () => {
+                    if (!liveness())
+                        return false;
+                    await store.refreshPage(local, key, composition, last
+                        ? { ...page, entries }
+                        : { entries, removals: [], folders: [], folderRemovals: [] });
+                    return true;
+                });
+                if (!staged)
+                    return false;
+                if (last)
+                    return true;
+                written = end;
+            }
+        },
+    };
+}
+export async function pullRefresh(installer, authorized, record, liveness, hold, refresh) {
+    if (!liveness())
+        return unconfirmed;
+    const { store, journal } = installer;
+    const from = standingAt(store, record.locator, record);
+    const local = await store.open();
+    try {
+        await store.exclusive(() => local.unrefresh(record.key));
+        const result = await advancement(async (authorization, request) => {
+            hold.release();
+            try {
+                return await refresh(authorization, request);
+            }
+            finally {
+                hold.take();
+            }
+        }, authorized, record, (models, kept) => (liveness() ? remodel(installer, models, kept) : kept), liveness, refreshingOf(installer, local, record.locator, record.key, store.composed(record), liveness, false, record.roomVersion));
+        if (result.kind !== 'advanced') {
+            journal.record(`transfer refresh ${record.locator} ${result.kind}`);
+            return result;
+        }
+        journal.record(`transfer refresh ${record.locator} ${result.record.roomVersion}`);
+        return {
+            kind: 'advanced',
+            record: result.record,
+            from,
+            remodelled: result.record.model !== record.model,
+        };
+    }
+    catch (error) {
+        if (refusedForSpace(error))
+            return unconfirmed;
+        journal.record(`transfer refresh ${record.locator} transport-uncertainty`);
+        throw error;
+    }
+}
+async function abandoned(installer, local, locator) {
+    await local.abandon(locator);
+    installer.store.installs.change(locator, (held) => transferGone(setBack(held)));
+    installer.recover();
+    installer.sweep();
+}
+function staging(installer, local, liveness, unheld) {
+    const { store, socket } = installer;
+    const derived = async (locator, key, composition) => {
+        store.installs.change(locator, (held) => ({ ...held, underived: 0 }));
+        for (;;) {
+            if (!liveness())
+                return false;
+            const done = await store.finishingWrite(async () => liveness() ? store.rederive(local, key, composition) : null);
+            if (done === null)
+                return false;
+            store.installs.change(locator, (held) => built(held, done.done, done.text));
+            installer.queueReadiness();
+            if (done.done)
+                return true;
+        }
+    };
+    return {
+        held: (locator) => store.installs.of(locator).transfer,
+        async turn(locator, step) {
+            installing(installer, locator);
+            return step === 'downloading' ? socket.heldStream() : () => { };
+        },
+        unqueued: socket.unqueued,
+        queued: socket.queued,
+        arriving(locator) {
+            let ended = false;
+            return {
+                at(bytes) {
+                    if (ended)
+                        return;
+                    store.installs.change(locator, (held) => ({
+                        ...held,
+                        arrived: bytes,
+                        streaming: true,
+                    }));
+                    if (!store.installs.arrivalDue(performance.now(), ARRIVAL_REPORT_MS))
+                        return;
+                    installer.queueReadiness();
+                },
+                end() {
+                    if (ended)
+                        return;
+                    ended = true;
+                    if (!store.installs.of(locator).streaming)
+                        return;
+                    store.installs.change(locator, (held) => ({ ...held, streaming: false }));
+                    installer.queueReadiness();
+                },
+            };
+        },
+        streamed: socket.streamed,
+        async stage(record, page, composition) {
+            const { locator } = record;
+            installing(installer, locator);
+            if (!liveness())
+                return null;
+            const key = await store.startingWrite(async () => {
+                if (!liveness())
+                    return null;
+                return local.stage(record, composition, page);
+            });
+            if (key === null || !liveness())
+                return null;
+            const before = store.installs.of(locator);
+            const prior = before.transfer?.done.bytes ?? 0;
+            const held = { ...record, key };
+            const text = page.entries.reduce((sum, entry) => sum + (entry.content?.length ?? 0), 0);
+            const owed = text + before.underived;
+            store.installs.change(locator, (room) => ({
+                ...owedIndexes(room, record.done.bytes - prior, text),
+                transfer: held,
+                underived: owed,
+            }));
+            if (before.transfer === null)
+                installer.recover();
+            installing(installer, locator);
+            installer.recover();
+            if (owed < store.deriveBytes())
+                return held;
+            return (await derived(locator, key, composition)) ? held : null;
+        },
+        async resume(record, parts, composition) {
+            const { locator } = record;
+            installing(installer, locator);
+            if (!liveness())
+                return null;
+            const prior = unheld === undefined ? await store.record(locator) : undefined;
+            const published = prior !== undefined && prior.replica.selection === record.selection ? prior.key : null;
+            const kept = unheld?.staged ?? (await local.held()).staged.has(locator);
+            if (published === null && !kept)
+                return { record: null, staged: new Set() };
+            const resumed = await store.startingWrite(async () => {
+                if (!liveness())
+                    return null;
+                return local.resume(record, composition, parts.map((part) => part.id), published);
+            });
+            if (resumed === null || !liveness())
+                return null;
+            const parted = new Set(resumed.staged);
+            if (resumed.key === null)
+                return { record: null, staged: parted };
+            const bytes = parts.reduce((sum, part) => sum + (parted.has(part.id) ? part.bytes : 0), 0);
+            const held = {
+                ...record,
+                transfer: resumed.transfer,
+                key: resumed.key,
+                done: { bytes },
+            };
+            store.installs.change(locator, (room) => ({ ...room, transfer: held }));
+            installer.recover();
+            return { record: held, staged: parted };
+        },
+        derive: derived,
+        async refreshing(locator, key, composition) {
+            await store.exclusive(() => local.unrefresh(key));
+            return refreshingOf(installer, local, locator, key, composition, liveness, true);
+        },
+        abandon: (locator) => store.exclusive(async () => {
+            if (liveness())
+                await abandoned(installer, local, locator);
+        }),
+        setAside: async (locator) => {
+            if (!liveness())
+                return;
+            store.installs.change(locator, (held) => transferGone(setBack(held)));
+            installer.recover();
+        },
+        release: (key) => store.exclusive(async () => {
+            await local.release(key);
+            installer.sweep();
+        }),
+    };
+}
+export async function pullBootstrap(installer, authorized, room, local, liveness, prior, from = installer.socket.reads, unheld) {
+    if (!liveness())
+        return unconfirmed;
+    const { store, journal } = installer;
+    if (prior !== undefined) {
+        catching(store, room.locator);
+        installer.recover();
+    }
+    try {
+        const result = await bootstrapRoom(from, authorized, room, { collection: prior?.model ?? installer.unheldModel(room.locator) ?? NO_MODEL }, installer.builtIn, installer.typedPaths, (models, held) => (liveness() ? remodel(installer, models, held) : held), liveness, staging(installer, local, liveness, unheld));
+        const detail = result.kind === 'staged' ? result.record.roomVersion : result.kind;
+        journal.record(`transfer bootstrap ${room.locator} ${detail}`);
+        if (result.kind === 'staged' &&
+            (await older(installer, room.locator, result.record.roomVersion))) {
+            caughtUpTo(installer, room.locator);
+            installed(installer, room.locator);
+            await abandoned(installer, local, room.locator);
+            journal.record(`transfer bootstrap ${room.locator} older than shown`);
+            return { kind: 'transport-uncertainty' };
+        }
+        if (result.kind !== 'staged') {
+            caughtUpTo(installer, room.locator);
+            installed(installer, room.locator);
+        }
+        return result;
+    }
+    catch (error) {
+        caughtUpTo(installer, room.locator);
+        installed(installer, room.locator);
+        if (refusedForSpace(error))
+            return unconfirmed;
+        journal.record(`transfer bootstrap ${room.locator} transport-uncertainty`);
+        throw error;
+    }
+}
+export function publish(installer, local, room, reached, landing, live) {
+    const { store } = installer;
+    const landed = store.exclusive(async () => {
+        if (!live()) {
+            caughtUpTo(installer, room.locator);
+            return true;
+        }
+        const work = await store.work(room.locator);
+        const prior = work.replica;
+        if (!standsAt(store, room.locator, prior?.record, landing.from)) {
+            if (landing.kind === 'refreshed') {
+                await local.unrefresh(reached.key);
+                return false;
+            }
+            caughtUpTo(installer, room.locator);
+            await abandoned(installer, local, room.locator);
+            return false;
+        }
+        const record = landing.kind === 'refreshed' && prior !== undefined
+            ? {
+                ...prior.record,
+                roomVersion: reached.roomVersion,
+                ...(landing.remodelled ? { model: reached.model } : {}),
+            }
+            : reached;
+        let set;
+        try {
+            set = await installer.settle(local, room.locator, work, record, work.queue, landing.kind === 'refreshed'
+                ? {
+                    room: true,
+                    view: 'refreshed',
+                    commit: { refreshed: true },
+                }
+                : {
+                    room: true,
+                    view: landing.refreshed ? 'refreshed' : 'confirmed',
+                    commit: {
+                        published: { from: prior?.record.key ?? null },
+                        ...(landing.refreshed ? { refreshed: true } : {}),
+                    },
+                });
+        }
+        catch (error) {
+            if (!refusedForSpace(error))
+                throw error;
+            caughtUpTo(installer, room.locator);
+            return true;
+        }
+        if (landing.kind === 'bootstrapped') {
+            installer.readied(room.locator);
+            store.installs.change(room.locator, (held) => ({
+                ...transferGone(held),
+                highest: null,
+                beyondHorizon: false,
+                startedOver: null,
+                catching: false,
+            }));
+            installer.modelKept(room.locator);
+            if (prior !== undefined && prior.record.key !== record.key)
+                installer.sweep();
+            void installer.name(room.locator);
+        }
+        installer.usable(room);
+        void installer.publishChangeSet(set);
+        await installer.unpair(local);
+        void installer.statusShown();
+        if (landing.kind === 'bootstrapped' && work.queue.length > 0) {
+            installer.submitLater(room.locator);
+        }
+        return true;
+    });
+    return landing.kind === 'refreshed'
+        ? landed
+        : landed.finally(() => installed(installer, room.locator));
+}

package/dist/job.d.ts

@@ -0,0 +1,33 @@
+type Job = {
+    readonly label: string;
+    readonly kind: 'started' | 'scheduled' | 'ended';
+    readonly state: string;
+    readonly next: string | null;
+};
+export type RunningJob = {
+    readonly name: string;
+    readonly running_since: string;
+};
+export type JobMarker = {
+    readonly said: string;
+    readonly at: string;
+};
+export type OpenJob<At = string | null> = {
+    readonly name: string;
+    readonly started: At;
+};
+export declare function jobOf(said: string): Job;
+export declare function closeLast(open: {
+    readonly key: string;
+}[], key: string): void;
+export declare function markerMoved<Kept extends {
+    readonly key: string;
+}>(open: Kept[], said: string, opened: (job: Job, key: string) => Kept | null): void;
+export declare function openJobs<At>(markers: readonly {
+    readonly said: string;
+    readonly at: At;
+}[]): readonly OpenJob<At>[];
+export declare function jobsRunning(jobs: readonly RunningJob[], now: number): string[];
+export declare function sessionEnded(jobs: readonly RunningJob[], now: number): string;
+export {};
+//# sourceMappingURL=job.d.ts.map

package/dist/job.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job.d.ts","sourceRoot":"","sources":["../src/job.ts"],"names":[],"mappings":"AAAA,KAAK,GAAG,GAAG;IACT,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,OAAO,CAAC;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE,MAAM,MAAM,OAAO,CAAC,EAAE,GAAG,MAAM,GAAG,IAAI,IAAI;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAA;CAAE,CAAC;AAQ1F,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CASvC;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAG7E;AAED,wBAAgB,WAAW,CAAC,IAAI,SAAS;IAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EAC/D,IAAI,EAAE,IAAI,EAAE,EACZ,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI,GAC7C,IAAI,CASN;AAED,wBAAgB,QAAQ,CAAC,EAAE,EACzB,OAAO,EAAE,SAAS;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAA;CAAE,EAAE,GAC7D,SAAS,OAAO,CAAC,EAAE,CAAC,EAAE,CAQxB;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAM9E;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAI7E"}

package/dist/job.js

@@ -0,0 +1,54 @@
+const NEXT_RUN = 'next run ';
+const MINUTE = 60_000;
+const ENDED = 'Session ended.';
+export function jobOf(said) {
+    const at = said.lastIndexOf(': ');
+    const label = at === -1 ? said : said.slice(0, at);
+    const state = at === -1 ? '' : said.slice(at + 2);
+    if (state === 'running')
+        return { label, kind: 'started', state, next: null };
+    if (state === 'scheduled')
+        return { label, kind: 'scheduled', state, next: null };
+    return state.startsWith(NEXT_RUN)
+        ? { label, kind: 'scheduled', state, next: state.slice(NEXT_RUN.length) }
+        : { label, kind: 'ended', state, next: null };
+}
+export function closeLast(open, key) {
+    const at = open.map((one) => one.key).lastIndexOf(key);
+    if (at !== -1)
+        open.splice(at, 1);
+}
+export function markerMoved(open, said, opened) {
+    const job = jobOf(said);
+    const started = `job\u0000${job.label}`;
+    const schedule = `schedule\u0000${job.label}`;
+    if (job.kind === 'ended')
+        closeLast(open, started);
+    if (job.kind !== 'started')
+        closeLast(open, schedule);
+    if (job.kind === 'ended')
+        return;
+    const kept = opened(job, job.kind === 'started' ? started : schedule);
+    if (kept !== null)
+        open.push(kept);
+}
+export function openJobs(markers) {
+    const open = [];
+    for (const { said, at } of markers) {
+        markerMoved(open, said, (job, key) => job.kind === 'started' ? { key, name: job.label, started: at } : null);
+    }
+    return open.map(({ name, started }) => ({ name, started }));
+}
+export function jobsRunning(jobs, now) {
+    return jobs.map((job) => {
+        const since = Date.parse(job.running_since);
+        const ran = Number.isNaN(since) ? 0 : Math.floor((now - since) / MINUTE);
+        return ran >= 1 ? `${job.name} (${ran} min)` : job.name;
+    });
+}
+export function sessionEnded(jobs, now) {
+    if (jobs.length === 0)
+        return ENDED;
+    const counted = jobs.length === 1 ? '1 background job' : `${jobs.length} background jobs`;
+    return `${ENDED} ${counted} stopped: ${jobsRunning(jobs, now).join(', ')}`;
+}

package/dist/live.d.ts

@@ -1,6 +1,7 @@
 import type { ChangeSet } from './change-set.js';
 import { type KeyedResult, type QueryMessage } from './observation.js';
 import type { PendingResult, Plan, QueryCoverage } from './query.js';
+import type { Fail } from './reach.js';
 import type { QueryRoom } from './tables.js';
 export type Cover = {
     coverage: QueryCoverage;
@@ -9,14 +10,14 @@ export type Cover = {
 export type Ran = KeyedResult & {
     readonly current: () => boolean;
 };
-export type Run = (plan: Plan, cover: () => Cover) => Promise<Ran>;
+export type Run = (plan: Plan, cover: () => Promise<Cover>) => Promise<Ran>;
 type Listener = (message: QueryMessage) => void;
-export type LiveQueries = {
+export type LiveQueries<H> = {
     touched(set: ChangeSet): void;
     recover(): void;
-    watch(plan: Plan, identity: string, cover: () => Cover, deliver: Listener): () => void;
+    watch(plan: Plan, identity: string, cover: (held: H) => Cover, deliver: Listener, failed: Fail): () => void;
     close(): void;
 };
-export declare function createLiveQueries(run: Run): LiveQueries;
+export declare function createLiveQueries<H>(run: Run, holding: () => Promise<H>): LiveQueries<H>;
 export {};
 //# sourceMappingURL=live.d.ts.map

package/dist/live.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"live.d.ts","sourceRoot":"","sources":["../src/live.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAIL,KAAK,WAAW,EAChB,KAAK,YAAY,EAClB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAErE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,MAAM,MAAM,KAAK,GAAG;IAAE,QAAQ,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,SAAS,SAAS,EAAE,CAAA;CAAE,GAAG,aAAa,CAAC;AAE7F,MAAM,MAAM,GAAG,GAAG,WAAW,GAAG;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,OAAO,CAAA;CAAE,CAAC;AAEpE,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,KAAK,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;AAEnE,KAAK,QAAQ,GAAG,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;AAchD,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B,OAAO,IAAI,IAAI,CAAC;IAChB,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,EAAE,OAAO,EAAE,QAAQ,GAAG,MAAM,IAAI,CAAC;IACvF,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAmCF,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,GAAG,GAAG,WAAW,CAuFvD"}
+{"version":3,"file":"live.d.ts","sourceRoot":"","sources":["../src/live.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAIL,KAAK,WAAW,EAChB,KAAK,YAAY,EAClB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AACrE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,MAAM,MAAM,KAAK,GAAG;IAAE,QAAQ,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,SAAS,SAAS,EAAE,CAAA;CAAE,GAAG,aAAa,CAAC;AAE7F,MAAM,MAAM,GAAG,GAAG,WAAW,GAAG;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,OAAO,CAAA;CAAE,CAAC;AAEpE,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;AAE5E,KAAK,QAAQ,GAAG,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;AAchD,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI;IAC3B,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B,OAAO,IAAI,IAAI,CAAC;IAChB,KAAK,CACH,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,KAAK,EACzB,OAAO,EAAE,QAAQ,EACjB,MAAM,EAAE,IAAI,GACX,MAAM,IAAI,CAAC;IACd,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAmCF,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CA4HxF"}