@agentshouse/mdruntime-adapter 0.1.0-cfd03209dcd82 → 0.1.0-cfded86738bb2

package/dist/stream.js

@@ -1,5 +1,7 @@
+import { ROW_WEIGHT, } from '@agentshouse/mdruntime';
 import { acceptedMessage } from './actions.js';
-import { binaryFileOf, headChange } from './read.js';
+import { sha256Hex } from './digest.js';
+import { answered, answering, binaryFileOf, folded, forgotten, headChange, revised, } from './read.js';
 import { roomVersion } from './version.js';
 const HELD_FRAMES_MAX = 256;
 function turnOperation(frame) {
@@ -14,6 +16,7 @@ function listedConversation(listed) {
         created: listed.created_at,
         finished: listed.finished_at,
         outcome: listed.outcome,
+        waiting: listed.waiting,
     };
 }
 function position(value) {
@@ -21,9 +24,78 @@ function position(value) {
         return null;
     return BigInt(value);
 }
+function assembled(entries) {
+    const whole = [];
+    let open = null;
+    for (const entry of entries) {
+        if (entry.kind === 'piece') {
+            if (open === null || open.entry.path !== entry.path || entry.text === undefined)
+                return null;
+            open.texts.push(entry.text);
+        }
+        else {
+            if (open !== null)
+                return null;
+            if (entry.continued !== true) {
+                whole.push(entry);
+                continue;
+            }
+            if (entry.text === undefined)
+                return null;
+            open = { entry, texts: [entry.text] };
+        }
+        if (entry.continued === true)
+            continue;
+        const { continued: _continued, ...head } = open.entry;
+        whole.push({ ...head, text: open.texts.join('') });
+        open = null;
+    }
+    return open === null ? whole : null;
+}
+async function verified(entries) {
+    try {
+        for (const entry of entries) {
+            if (entry.text === undefined || entry.sha256 === undefined)
+                continue;
+            if ((await sha256Hex(entry.text)) !== entry.sha256)
+                return false;
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function resumed(kept, frame) {
+    const last = frame.entries.at(-1);
+    const point = { parts: frame.index + 1, from: frame.from, target: frame.target };
+    if (last?.continued !== true)
+        return { ...point, handled: frame.handled, piece: null };
+    const text = last.text?.length ?? 0;
+    if (last.kind !== 'piece') {
+        return last.sha256 === undefined
+            ? kept.resume
+            : { ...point, handled: frame.handled, piece: { sha256: last.sha256, offset: text } };
+    }
+    const piece = kept.resume?.piece ?? null;
+    if (piece === null)
+        return kept.resume;
+    const pieces = frame.entries.reduce((sum, entry) => sum + (entry.kind === 'piece' ? (entry.text?.length ?? 0) : 0), 0);
+    return {
+        ...point,
+        handled: frame.handled,
+        piece: { sha256: piece.sha256, offset: piece.offset + pieces },
+    };
+}
 export function houseNotifications(transport, mode) {
     const stream = transport.stream;
     const lines = new Map();
+    const kept = new Map();
+    let receipts = null;
+    let loaded = Promise.resolve();
+    let acknowledging = null;
+    let receiving = Promise.resolve();
+    let verifying = 0;
     const locators = new Map();
     const observers = new Set();
     const conversationObservers = new Set();
@@ -41,7 +113,10 @@ export function houseNotifications(transport, mode) {
             event.kind === 'questions' ||
             event.kind === 'process' ||
             event.kind === 'commands' ||
-            event.kind === 'options') {
+            event.kind === 'options' ||
+            event.kind === 'starting' ||
+            event.kind === 'link' ||
+            event.kind === 'deliveries') {
             live.add(event.conversation);
         }
         for (const observer of conversationObservers)
@@ -49,6 +124,7 @@ export function houseNotifications(transport, mode) {
     }
     function reopened(next) {
         opening = next;
+        acknowledging = next;
         const gapped = new Set([...replicated, ...live]);
         live.clear();
         for (const conversation of gapped)
@@ -86,6 +162,18 @@ export function houseNotifications(transport, mode) {
             emitConversation({ kind: 'options', conversation, options: structuredClone(frame.options) });
             return;
         }
+        if (frame.type === 'starting') {
+            emitConversation({ kind: 'starting', conversation, step: frame.step });
+            return;
+        }
+        if (frame.type === 'link') {
+            emitConversation({ kind: 'link', conversation, connected: frame.connected });
+            return;
+        }
+        if (frame.type === 'deliveries') {
+            emitConversation({ kind: 'deliveries', conversation, unknown: [...frame.unknown] });
+            return;
+        }
         if (frame.type === 'interactions') {
             emitConversation({
                 kind: 'questions',
@@ -110,7 +198,162 @@ export function houseNotifications(transport, mode) {
         if (line !== undefined)
             lines.set(authority, { ...line, frames: [] });
     }
-    function receive(frame) {
+    async function forget(carries) {
+        const held = carries.filter((carry) => kept.delete(carry));
+        if (held.length > 0)
+            await receipts?.drop(held).catch(() => { });
+    }
+    async function lost(authority, carry) {
+        interrupt(authority);
+        await forget([carry]);
+    }
+    async function stage(frame, via) {
+        await loaded;
+        const port = receipts;
+        if (port === null)
+            return;
+        const held = kept.get(frame.carry);
+        if (held === undefined && frame.index !== 0) {
+            interrupt(frame.authority);
+            return;
+        }
+        if (held !== undefined &&
+            (held.fenced !== null || frame.index > held.parts || frame.index < (held.resume?.parts ?? 0))) {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        if (held === undefined) {
+            await forget([...kept.values()]
+                .filter((other) => other.authority === frame.authority && other.fenced === null)
+                .map((other) => other.carry));
+        }
+        const start = held ?? {
+            authority: frame.authority,
+            carry: frame.carry,
+            logEpoch: lines.get(frame.authority)?.logEpoch ?? null,
+            after: frame.after,
+            parts: 0,
+            resume: null,
+            fenced: null,
+        };
+        const next = { ...start, parts: frame.index + 1, resume: resumed(start, frame) };
+        const part = { entries: frame.entries };
+        try {
+            await port.keep(frame.carry, next, { index: frame.index, value: part });
+        }
+        catch {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        kept.set(frame.carry, next);
+        via?.received(frame.authority, frame.carry, frame.index);
+    }
+    async function keptEntries(carry, parts) {
+        if (parts === 0)
+            return [];
+        if (receipts === null)
+            return null;
+        const read = (await receipts.parts(carry, parts));
+        if (read.length !== parts)
+            return null;
+        const entries = assembled(read.flatMap((part) => part.entries));
+        return entries !== null && (await verified(entries)) ? entries : null;
+    }
+    async function carried(frame) {
+        await loaded;
+        const held = kept.get(frame.carry);
+        if (frame.parts > 0 && held?.parts !== frame.parts) {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        const entries = await keptEntries(frame.carry, frame.parts).catch(() => null);
+        if (entries === null) {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        const fenced = {
+            ...(held ?? {
+                authority: frame.authority,
+                carry: frame.carry,
+                logEpoch: frame.log_epoch,
+                after: frame.after,
+                parts: 0,
+                resume: null,
+            }),
+            after: frame.after,
+            fenced: { position: frame.position, logEpoch: frame.log_epoch },
+        };
+        try {
+            await receipts?.keep(frame.carry, fenced);
+        }
+        catch {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        kept.set(frame.carry, fenced);
+        publish(frame, entries);
+    }
+    async function load(port) {
+        try {
+            for (const { key, head } of await port.heads())
+                kept.set(key, head);
+        }
+        catch {
+            kept.clear();
+        }
+    }
+    async function restored(room) {
+        const line = lines.get(room.authority);
+        let at = room.position;
+        const chain = [];
+        for (;;) {
+            const next = [...kept.values()].find((carry) => carry.authority === room.authority &&
+                carry.after === at &&
+                carry.fenced?.logEpoch === room.log_epoch);
+            if (next?.fenced == null)
+                break;
+            const entries = await keptEntries(next.carry, next.parts).catch(() => null);
+            if (entries === null)
+                break;
+            chain.push({ carry: next, entries });
+            at = next.fenced.position;
+        }
+        const partial = [...kept.values()].find((carry) => carry.authority === room.authority &&
+            carry.fenced === null &&
+            carry.logEpoch === room.log_epoch &&
+            carry.after === at &&
+            carry.resume !== null &&
+            carry.resume.parts > 0);
+        const stale = [...kept.values()].filter((carry) => carry.authority === room.authority &&
+            carry !== partial &&
+            !chain.some((link) => link.carry === carry));
+        await forget(stale.map((carry) => carry.carry));
+        if (line !== undefined && line.logEpoch === room.log_epoch && chain.length > 0) {
+            const frames = chain.map(({ carry, entries }) => ({
+                after: BigInt(carry.after),
+                position: BigInt(carry.fenced.position),
+                entries,
+            }));
+            lines.set(room.authority, { ...line, frames: [...line.frames, ...frames] });
+        }
+        return {
+            ...room,
+            position: at,
+            ...(partial?.resume == null
+                ? {}
+                : {
+                    carry: {
+                        id: partial.carry,
+                        parts: partial.resume.parts,
+                        from: partial.resume.from,
+                        target: partial.resume.target,
+                        handled: partial.resume.handled,
+                        ...(partial.resume.piece === null ? {} : { piece: partial.resume.piece }),
+                    },
+                }),
+        };
+    }
+    function handle(frame, via) {
         if (frame.type === 'signal') {
             const mode = frame.service_mode;
             if (mode !== 'open' && mode !== 'write-halted')
@@ -126,6 +369,10 @@ export function houseNotifications(transport, mode) {
             });
             return;
         }
+        if (frame.type === 'authority_changed') {
+            emit({ kind: 'authority-changed', scope: { authorities: frame.rooms } });
+            return;
+        }
         if (frame.type === 'conversations') {
             emitConversation({ kind: 'list', entries: frame.conversations.map(listedConversation) });
             return;
@@ -142,13 +389,20 @@ export function houseNotifications(transport, mode) {
             frame.type === 'process' ||
             frame.type === 'commands' ||
             frame.type === 'options' ||
+            frame.type === 'starting' ||
+            frame.type === 'link' ||
+            frame.type === 'deliveries' ||
             frame.type === 'interactions' ||
             frame.type === 'message') {
             converse(frame);
             return;
         }
-        if (frame.type !== 'entries')
-            return;
+        if (frame.type === 'part')
+            return stage(frame, via);
+        if (frame.type === 'carried')
+            return carried(frame);
+    }
+    function publish(frame, entries) {
         const at = position(frame.position);
         const after = position(frame.after);
         if (at === null || after === null || after >= at) {
@@ -162,7 +416,7 @@ export function houseNotifications(transport, mode) {
             const last = line.frames.at(-1);
             if (last !== undefined && at <= last.position)
                 return;
-            const held = { after, position: at, entries: frame.entries };
+            const held = { after, position: at, entries };
             const next = last !== undefined && after <= last.position ? [...line.frames, held] : [held];
             lines.set(frame.authority, { ...line, frames: next.length > HELD_FRAMES_MAX ? [] : next });
         }
@@ -171,11 +425,30 @@ export function houseNotifications(transport, mode) {
         }
         announce(frame.authority, { position: frame.position, logEpoch: frame.log_epoch });
     }
-    stream.observe(receive);
+    stream.observe((frame) => {
+        const via = acknowledging;
+        if (verifying === 0) {
+            const handled = handle(frame, via);
+            if (handled === undefined)
+                return;
+            verifying += 1;
+            receiving = handled.finally(() => {
+                verifying -= 1;
+            });
+            return;
+        }
+        verifying += 1;
+        receiving = receiving
+            .then(() => handle(frame, via))
+            .finally(() => {
+            verifying -= 1;
+        });
+    });
     stream.opened(reopened);
-    stream.closed((revoked) => {
+    stream.closed(({ revoked, lost }) => {
         opening = null;
-        emit({ kind: 'closed', revoked });
+        acknowledging = null;
+        emit({ kind: 'closed', revoked, lost });
     });
     return {
         runtime,
@@ -198,8 +471,12 @@ export function houseNotifications(transport, mode) {
                 };
             },
         },
-        observe(notify) {
+        observe(notify, port) {
             observers.add(notify);
+            if (receipts === null) {
+                receipts = port;
+                loaded = load(port);
+            }
             if (opening !== null) {
                 queueMicrotask(() => {
                     if (observers.has(notify))
@@ -220,7 +497,30 @@ export function houseNotifications(transport, mode) {
             }
             const current = opening;
             opening = null;
-            current?.hold(rooms, runtime);
+            if (current === null)
+                return;
+            void loaded
+                .then(() => Promise.all(rooms.map(restored)))
+                .then((presented) => {
+                current.hold(presented, runtime);
+                for (const [index, room] of presented.entries()) {
+                    if (room.position === rooms[index].position)
+                        continue;
+                    announce(room.authority, { position: room.position, logEpoch: room.log_epoch });
+                }
+            });
+        },
+        published(locator, at) {
+            const authority = [...locators].find(([, held]) => held.includes(locator))?.[0];
+            const reached = position(at.position);
+            if (authority === undefined || reached === null)
+                return;
+            void forget([...kept.values()]
+                .filter((carry) => carry.authority === authority &&
+                carry.fenced !== null &&
+                carry.fenced.logEpoch === at.logEpoch &&
+                BigInt(carry.fenced.position) <= reached)
+                .map((carry) => carry.carry));
         },
         reopen() {
             const current = opening;
@@ -239,6 +539,9 @@ export function houseNotifications(transport, mode) {
                 if (!locators.has(authority))
                     lines.delete(authority);
             }
+            void loaded.then(() => forget([...kept.values()]
+                .filter((carry) => !locators.has(carry.authority))
+                .map((carry) => carry.carry)));
         },
         reach(authority, at) {
             const line = lines.get(authority);
@@ -253,7 +556,18 @@ export function houseNotifications(transport, mode) {
                 held,
             });
         },
-        advance(tracked, from, holds) {
+        edited(tracked, from) {
+            const line = lines.get(tracked.authority);
+            const start = position(from.position);
+            if (line === undefined || start === null || line.logEpoch !== from.logEpoch)
+                return [];
+            return [
+                ...new Set(line.frames
+                    .filter((frame) => frame.position > start)
+                    .flatMap((frame) => frame.entries.flatMap((entry) => entry.kind === 'edited' && entry.revision !== undefined ? [entry.path] : []))),
+            ];
+        },
+        *advance(tracked, from, holds) {
             const line = lines.get(tracked.authority);
             const start = position(from.position);
             if (line === undefined || start === null || line.logEpoch !== from.logEpoch)
@@ -262,36 +576,26 @@ export function houseNotifications(transport, mode) {
             const first = pending[0];
             if (first === undefined || first.after > start)
                 return null;
-            const documents = {};
-            const revisions = {};
-            const changes = {};
-            const binaryFiles = {};
-            const removals = [];
-            const applied = [];
+            const answer = answering();
             const schemas = new Map();
-            const forget = (path) => {
-                delete documents[path];
-                delete revisions[path];
-                delete changes[path];
-                delete binaryFiles[path];
-                const removed = removals.indexOf(path);
-                if (removed !== -1)
-                    removals.splice(removed, 1);
-            };
             for (const frame of pending) {
                 for (const entry of frame.entries) {
+                    yield ROW_WEIGHT + (entry.text?.length ?? 0);
+                    if (entry.kind === 'folder') {
+                        folded(answer, entry.path, entry.removed === true);
+                        continue;
+                    }
                     if (entry.kind === 'edited' &&
                         entry.revision !== undefined &&
                         holds?.(entry.path, entry.revision) === true) {
-                        forget(entry.path);
+                        forgotten(answer, entry.path);
                         continue;
                     }
-                    applied.push(entry);
                     if (tracked.schemas.includes(entry.path))
                         schemas.set(entry.path, entry);
                     if (entry.kind === 'removed') {
-                        forget(entry.path);
-                        removals.push(entry.path);
+                        forgotten(answer, entry.path);
+                        answer.removals.add(entry.path);
                         continue;
                     }
                     const binary = binaryFileOf(entry);
@@ -302,31 +606,33 @@ export function houseNotifications(transport, mode) {
                     const head = headChange(entry);
                     if (head === null)
                         return null;
-                    forget(entry.path);
+                    forgotten(answer, entry.path);
                     if (binary === undefined)
-                        documents[entry.path] = entry.text;
+                        answer.documents.set(entry.path, entry.text);
                     else
-                        binaryFiles[entry.path] = binary;
+                        answer.binaryFiles.set(entry.path, binary);
                     const revision = entry.revision ?? binary?.sha256;
                     if (revision !== undefined)
-                        revisions[entry.path] = revision;
+                        revised(answer, entry.path, revision);
                     if (head !== undefined)
-                        changes[entry.path] = head;
+                        answer.changes.set(entry.path, head);
                 }
             }
             const last = pending.at(-1);
-            lines.set(tracked.authority, { ...line, frames: pending, held: last.position });
+            const current = lines.get(tracked.authority);
+            if (current?.logEpoch === line.logEpoch) {
+                lines.set(tracked.authority, {
+                    ...current,
+                    frames: current.frames.filter((frame) => frame.position > start),
+                    held: current.held !== null && current.held > last.position ? current.held : last.position,
+                });
+            }
             return {
                 page: {
                     kind: 'net-advancement',
                     roomVersion: roomVersion({ position: last.position.toString(), logEpoch: line.logEpoch }),
-                    documents,
-                    revisions,
-                    changes,
-                    binaryFiles,
-                    removals: removals.sort(),
+                    ...(yield* answered(answer)),
                 },
-                entries: applied,
                 schemas,
             };
         },

package/dist/transport.d.ts

@@ -15,27 +15,26 @@ export type HouseDoorAnswer = {
     body: unknown;
 };
 export declare const DOOR_FILES_GET = "/door/files/get";
+export declare const DOOR_FILES_FINISH = "/door/files/finish";
 export declare const SITE_FILES = "/_house/files/";
-export declare const SITE_FILE_RECORD = "record";
-export type HouseByteAct = 'authored_batch_upload' | 'conversation_file_upload' | 'binary_stage';
-export type HouseByteBody = string | ArrayBuffer | Uint8Array | Blob;
+export type HouseByteAct = 'authored_batch_upload';
 export type HouseBytesAnswer = {
     status: number;
-    blob: Blob | null;
+    body: ReadableStream<Uint8Array> | null;
+    mediaType: string;
 };
 export type HouseByteTransfer = {
     act: HouseByteAct;
     url: string;
-    body: HouseByteBody;
-    operation?: string;
+    body: string;
 };
 export type HouseStreamEntry = {
     path: string;
-    kind: 'edited' | 'removed';
+    kind: 'edited' | 'removed' | 'piece' | 'folder';
+    removed?: true;
     revision?: string;
     sha256?: string;
     text?: string;
-    file?: string;
     bytes?: number;
     media_type?: string;
     author?: {
@@ -46,6 +45,7 @@ export type HouseStreamEntry = {
         name: string;
     };
     created_at?: string;
+    continued?: true;
 };
 type LiveFact<K extends ConversationEvent['kind']> = Extract<ConversationEvent, {
     kind: K;
@@ -57,17 +57,32 @@ export type HouseListedConversation = {
     created_at: string;
     finished_at: string | null;
     outcome: string | null;
+    waiting: boolean;
 };
 export type HouseStreamFrame = {
-    type: 'entries';
+    type: 'part';
     authority: string;
+    carry: string;
+    index: number;
+    after: string;
+    from: string;
+    target: string;
+    handled: number;
+    entries: readonly HouseStreamEntry[];
+} | {
+    type: 'carried';
+    authority: string;
+    carry: string;
+    parts: number;
     after: string;
     position: string;
     log_epoch: string;
-    entries: readonly HouseStreamEntry[];
 } | {
     type: 'opening';
     expired: readonly string[];
+} | {
+    type: 'authority_changed';
+    rooms: readonly string[];
 } | ({
     type: 'turn';
     conversation_id: string;
@@ -80,6 +95,10 @@ export type HouseStreamFrame = {
     type: 'interactions';
     conversation_id: string;
     interactions: readonly unknown[];
+} | {
+    type: 'deliveries';
+    conversation_id: string;
+    unknown: readonly string[];
 } | {
     type: 'process';
     conversation_id: string;
@@ -92,6 +111,14 @@ export type HouseStreamFrame = {
     type: 'options';
     conversation_id: string;
     options: LiveFact<'options'>['options'];
+} | {
+    type: 'starting';
+    conversation_id: string;
+    step: string | null;
+} | {
+    type: 'link';
+    conversation_id: string;
+    connected: boolean;
 } | {
     type: 'conversations';
     conversations: readonly HouseListedConversation[];
@@ -105,31 +132,49 @@ export type HouseStreamFrame = {
     type: 'signal';
     service_mode: string;
 };
+export type HouseHeldCarry = {
+    id: string;
+    parts: number;
+    from: string;
+    target: string;
+    handled: number;
+    piece?: {
+        sha256: string;
+        offset: number;
+    };
+};
 export type HouseHeldRoom = {
     authority: string;
     position: string;
     log_epoch: string;
+    carry?: HouseHeldCarry;
 };
 export type HouseOpening = {
     hold(rooms: readonly HouseHeldRoom[], runtime: string): void;
+    received(authority: string, carry: string, index: number): void;
     close(): void;
 };
+export type HouseClosing = {
+    revoked: boolean;
+    lost: boolean;
+};
 export type HouseStream = {
     observe(receive: (frame: HouseStreamFrame) => void): () => void;
     opened(notify: (opening: HouseOpening) => void): () => void;
-    closed(notify: (revoked: boolean) => void): () => void;
+    closed(notify: (closing: HouseClosing) => void): () => void;
 };
 export type HouseDoor = {
-    door(route: string, body: unknown): Promise<HouseDoorAnswer>;
+    door(route: string, body: unknown, arriving?: (bytes: number) => void): Promise<HouseDoorAnswer>;
     stream: HouseStream;
 };
 export type HouseTransport = HouseDoor & {
     transfer(request: HouseByteTransfer): Promise<HouseDoorAnswer>;
+    put(url: string, body: Blob): Promise<number>;
     download(url: string): Promise<HouseBytesAnswer>;
 };
 export type HouseSiteTransport = HouseDoor & {
     online(): boolean;
-    read(route: string): Promise<HouseDoorAnswer>;
+    read(route: string, arriving?: (bytes: number) => void): Promise<HouseDoorAnswer>;
     download(route: string): Promise<HouseBytesAnswer>;
 };
 export {};