@agentshouse/mdruntime-adapter 0.1.0-cc63dd043d0e2 → 0.1.0-cc710a8825617

package/dist/stream.js

@@ -1,45 +1,130 @@
 import { acceptedMessage } from './actions.js';
-import { headChange } from './read.js';
+import { sha256Hex } from './digest.js';
+import { binaryFileOf, headChange, identities } from './read.js';
 import { roomVersion } from './version.js';
 const HELD_FRAMES_MAX = 256;
 function turnOperation(frame) {
     const { type: _type, conversation_id: _conversation, turn_id: _turn, ...operation } = frame;
     return structuredClone(operation);
 }
+function listedConversation(listed) {
+    return {
+        conversation: listed.conversation_id,
+        agent: listed.agent_id,
+        name: listed.name,
+        created: listed.created_at,
+        finished: listed.finished_at,
+        outcome: listed.outcome,
+    };
+}
 function position(value) {
     if (!/^(?:0|[1-9][0-9]*)$/.test(value))
         return null;
     return BigInt(value);
 }
+function assembled(entries) {
+    const whole = [];
+    let open = null;
+    for (const entry of entries) {
+        if (entry.kind === 'piece') {
+            if (open === null || open.entry.path !== entry.path || entry.text === undefined)
+                return null;
+            open.texts.push(entry.text);
+        }
+        else {
+            if (open !== null)
+                return null;
+            if (entry.continued !== true) {
+                whole.push(entry);
+                continue;
+            }
+            if (entry.text === undefined)
+                return null;
+            open = { entry, texts: [entry.text] };
+        }
+        if (entry.continued === true)
+            continue;
+        const { continued: _continued, ...head } = open.entry;
+        whole.push({ ...head, text: open.texts.join('') });
+        open = null;
+    }
+    return open === null ? whole : null;
+}
+async function verified(entries) {
+    try {
+        for (const entry of entries) {
+            if (entry.text === undefined || entry.sha256 === undefined)
+                continue;
+            if ((await sha256Hex(entry.text)) !== entry.sha256)
+                return false;
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function resumed(kept, frame) {
+    const last = frame.entries.at(-1);
+    const point = { parts: frame.index + 1, from: frame.from, target: frame.target };
+    if (last?.continued !== true)
+        return { ...point, handled: frame.handled, piece: null };
+    const text = last.text?.length ?? 0;
+    if (last.kind !== 'piece') {
+        return last.sha256 === undefined
+            ? kept.resume
+            : { ...point, handled: frame.handled, piece: { sha256: last.sha256, offset: text } };
+    }
+    const piece = kept.resume?.piece ?? null;
+    if (piece === null)
+        return kept.resume;
+    const pieces = frame.entries.reduce((sum, entry) => sum + (entry.kind === 'piece' ? (entry.text?.length ?? 0) : 0), 0);
+    return {
+        ...point,
+        handled: frame.handled,
+        piece: { sha256: piece.sha256, offset: piece.offset + pieces },
+    };
+}
 export function houseNotifications(transport, mode) {
     const stream = transport.stream;
     const lines = new Map();
+    const kept = new Map();
+    let receipts = null;
+    let loaded = Promise.resolve();
+    let acknowledging = null;
+    let receiving = Promise.resolve();
+    let verifying = 0;
     const locators = new Map();
-    const schemas = new Map();
     const observers = new Set();
     const conversationObservers = new Set();
     const modeObservers = new Set();
     let replicated = [];
     const live = new Set();
-    let listening = false;
-    let catching = false;
-    function listen() {
-        if (listening)
-            return;
-        listening = true;
-        stream.observe(receive);
+    let opening = null;
+    const runtime = crypto.randomUUID();
+    function emit(event) {
+        for (const observer of observers)
+            observer(event);
     }
     function emitConversation(event) {
-        if (event.kind !== 'gap' && event.kind !== 'accepted-message')
+        if (event.kind === 'turn' ||
+            event.kind === 'questions' ||
+            event.kind === 'process' ||
+            event.kind === 'commands' ||
+            event.kind === 'options') {
             live.add(event.conversation);
+        }
         for (const observer of conversationObservers)
             observer(event);
     }
-    function reopened() {
+    function reopened(next) {
+        opening = next;
+        acknowledging = next;
         const gapped = new Set([...replicated, ...live]);
         live.clear();
         for (const conversation of gapped)
             emitConversation({ kind: 'gap', conversation });
+        emit({ kind: 'opened' });
     }
     function converse(frame) {
         const conversation = frame.conversation_id;
@@ -85,14 +170,10 @@ export function houseNotifications(transport, mode) {
             return;
         emitConversation({ kind: 'accepted-message', conversation, message });
     }
-    function announce(authority, at, modelChanged) {
+    function announce(authority, at) {
         const version = roomVersion(at);
         for (const locator of locators.get(authority) ?? []) {
-            for (const observer of observers) {
-                observer(modelChanged
-                    ? { locator, roomVersion: version, modelChanged: true }
-                    : { locator, roomVersion: version });
-            }
+            emit({ kind: 'room-version', locator, roomVersion: version });
         }
     }
     function interrupt(authority) {
@@ -100,7 +181,162 @@ export function houseNotifications(transport, mode) {
         if (line !== undefined)
             lines.set(authority, { ...line, frames: [] });
     }
-    function receive(frame) {
+    async function forget(carries) {
+        const held = carries.filter((carry) => kept.delete(carry));
+        if (held.length > 0)
+            await receipts?.drop(held).catch(() => { });
+    }
+    async function lost(authority, carry) {
+        interrupt(authority);
+        await forget([carry]);
+    }
+    async function stage(frame, via) {
+        await loaded;
+        const port = receipts;
+        if (port === null)
+            return;
+        const held = kept.get(frame.carry);
+        if (held === undefined && frame.index !== 0) {
+            interrupt(frame.authority);
+            return;
+        }
+        if (held !== undefined &&
+            (held.fenced !== null || frame.index > held.parts || frame.index < (held.resume?.parts ?? 0))) {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        if (held === undefined) {
+            await forget([...kept.values()]
+                .filter((other) => other.authority === frame.authority && other.fenced === null)
+                .map((other) => other.carry));
+        }
+        const start = held ?? {
+            authority: frame.authority,
+            carry: frame.carry,
+            logEpoch: lines.get(frame.authority)?.logEpoch ?? null,
+            after: frame.after,
+            parts: 0,
+            resume: null,
+            fenced: null,
+        };
+        const next = { ...start, parts: frame.index + 1, resume: resumed(start, frame) };
+        const part = { entries: frame.entries };
+        try {
+            await port.keep(frame.carry, next, { index: frame.index, value: part });
+        }
+        catch {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        kept.set(frame.carry, next);
+        via?.received(frame.authority, frame.carry, frame.index);
+    }
+    async function keptEntries(carry, parts) {
+        if (parts === 0)
+            return [];
+        if (receipts === null)
+            return null;
+        const read = (await receipts.parts(carry, parts));
+        if (read.length !== parts)
+            return null;
+        const entries = assembled(read.flatMap((part) => part.entries));
+        return entries !== null && (await verified(entries)) ? entries : null;
+    }
+    async function carried(frame) {
+        await loaded;
+        const held = kept.get(frame.carry);
+        if (frame.parts > 0 && held?.parts !== frame.parts) {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        const entries = await keptEntries(frame.carry, frame.parts).catch(() => null);
+        if (entries === null) {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        const fenced = {
+            ...(held ?? {
+                authority: frame.authority,
+                carry: frame.carry,
+                logEpoch: frame.log_epoch,
+                after: frame.after,
+                parts: 0,
+                resume: null,
+            }),
+            after: frame.after,
+            fenced: { position: frame.position, logEpoch: frame.log_epoch },
+        };
+        try {
+            await receipts?.keep(frame.carry, fenced);
+        }
+        catch {
+            await lost(frame.authority, frame.carry);
+            return;
+        }
+        kept.set(frame.carry, fenced);
+        publish(frame, entries);
+    }
+    async function load(port) {
+        try {
+            for (const { key, head } of await port.heads())
+                kept.set(key, head);
+        }
+        catch {
+            kept.clear();
+        }
+    }
+    async function restored(room) {
+        const line = lines.get(room.authority);
+        let at = room.position;
+        const chain = [];
+        for (;;) {
+            const next = [...kept.values()].find((carry) => carry.authority === room.authority &&
+                carry.after === at &&
+                carry.fenced?.logEpoch === room.log_epoch);
+            if (next?.fenced == null)
+                break;
+            const entries = await keptEntries(next.carry, next.parts).catch(() => null);
+            if (entries === null)
+                break;
+            chain.push({ carry: next, entries });
+            at = next.fenced.position;
+        }
+        const partial = [...kept.values()].find((carry) => carry.authority === room.authority &&
+            carry.fenced === null &&
+            carry.logEpoch === room.log_epoch &&
+            carry.after === at &&
+            carry.resume !== null &&
+            carry.resume.parts > 0);
+        const stale = [...kept.values()].filter((carry) => carry.authority === room.authority &&
+            carry !== partial &&
+            !chain.some((link) => link.carry === carry));
+        await forget(stale.map((carry) => carry.carry));
+        if (line !== undefined && line.logEpoch === room.log_epoch && chain.length > 0) {
+            const frames = chain.map(({ carry, entries }) => ({
+                after: BigInt(carry.after),
+                position: BigInt(carry.fenced.position),
+                entries,
+            }));
+            lines.set(room.authority, { ...line, frames: [...line.frames, ...frames] });
+        }
+        return {
+            ...room,
+            position: at,
+            ...(partial?.resume == null
+                ? {}
+                : {
+                    carry: {
+                        id: partial.carry,
+                        parts: partial.resume.parts,
+                        from: partial.resume.from,
+                        target: partial.resume.target,
+                        handled: partial.resume.handled,
+                        ...(partial.resume.piece === null ? {} : { piece: partial.resume.piece }),
+                    },
+                }),
+        };
+    }
+    function handle(frame, via) {
         if (frame.type === 'signal') {
             const mode = frame.service_mode;
             if (mode !== 'open' && mode !== 'write-halted')
@@ -109,6 +345,25 @@ export function houseNotifications(transport, mode) {
                 observer(mode);
             return;
         }
+        if (frame.type === 'opening') {
+            emit({
+                kind: 'opening',
+                expired: frame.expired.flatMap((authority) => locators.get(authority) ?? []),
+            });
+            return;
+        }
+        if (frame.type === 'conversations') {
+            emitConversation({ kind: 'list', entries: frame.conversations.map(listedConversation) });
+            return;
+        }
+        if (frame.type === 'conversation') {
+            emitConversation({ kind: 'listed', entry: listedConversation(frame.conversation) });
+            return;
+        }
+        if (frame.type === 'conversation_deleted') {
+            emitConversation({ kind: 'unlisted', conversation: frame.conversation_id });
+            return;
+        }
         if (frame.type === 'turn' ||
             frame.type === 'process' ||
             frame.type === 'commands' ||
@@ -118,8 +373,12 @@ export function houseNotifications(transport, mode) {
             converse(frame);
             return;
         }
-        if (frame.type !== 'entries')
-            return;
+        if (frame.type === 'part')
+            return stage(frame, via);
+        if (frame.type === 'carried')
+            return carried(frame);
+    }
+    function publish(frame, entries) {
         const at = position(frame.position);
         const after = position(frame.after);
         if (at === null || after === null || after >= at) {
@@ -133,21 +392,45 @@ export function houseNotifications(transport, mode) {
             const last = line.frames.at(-1);
             if (last !== undefined && at <= last.position)
                 return;
-            const held = { after, position: at, entries: frame.entries };
+            const held = { after, position: at, entries };
             const next = last !== undefined && after <= last.position ? [...line.frames, held] : [held];
             lines.set(frame.authority, { ...line, frames: next.length > HELD_FRAMES_MAX ? [] : next });
         }
         else {
             interrupt(frame.authority);
         }
-        const addresses = schemas.get(frame.authority) ?? [];
-        announce(frame.authority, { position: frame.position, logEpoch: frame.log_epoch }, frame.entries.some((entry) => addresses.includes(entry.path)));
+        announce(frame.authority, { position: frame.position, logEpoch: frame.log_epoch });
     }
+    stream.observe((frame) => {
+        const via = acknowledging;
+        if (verifying === 0) {
+            const handled = handle(frame, via);
+            if (handled === undefined)
+                return;
+            verifying += 1;
+            receiving = handled.finally(() => {
+                verifying -= 1;
+            });
+            return;
+        }
+        verifying += 1;
+        receiving = receiving
+            .then(() => handle(frame, via))
+            .finally(() => {
+            verifying -= 1;
+        });
+    });
+    stream.opened(reopened);
+    stream.closed((revoked) => {
+        opening = null;
+        acknowledging = null;
+        emit({ kind: 'closed', revoked });
+    });
     return {
+        runtime,
         serviceMode: {
             observe(notify) {
                 modeObservers.add(notify);
-                listen();
                 return () => {
                     modeObservers.delete(notify);
                 };
@@ -159,22 +442,71 @@ export function houseNotifications(transport, mode) {
             },
             observe(notify) {
                 conversationObservers.add(notify);
-                listen();
-                if (!catching) {
-                    catching = true;
-                    stream.opened(reopened);
-                }
                 return () => {
                     conversationObservers.delete(notify);
                 };
             },
         },
+        observe(notify, port) {
+            observers.add(notify);
+            if (receipts === null) {
+                receipts = port;
+                loaded = load(port);
+            }
+            if (opening !== null) {
+                queueMicrotask(() => {
+                    if (observers.has(notify))
+                        notify({ kind: 'opened' });
+                });
+            }
+            return () => {
+                observers.delete(notify);
+            };
+        },
+        send(rooms, demand) {
+            for (const room of rooms) {
+                const line = lines.get(room.authority);
+                const held = position(room.position);
+                if (line === undefined || held === null || line.logEpoch !== room.log_epoch)
+                    continue;
+                lines.set(room.authority, { ...line, frames: [], held });
+            }
+            const current = opening;
+            opening = null;
+            if (current === null)
+                return;
+            void loaded
+                .then(() => Promise.all(rooms.map(restored)))
+                .then((presented) => {
+                current.hold(presented, runtime, demand);
+                for (const [index, room] of presented.entries()) {
+                    if (room.position === rooms[index].position)
+                        continue;
+                    announce(room.authority, { position: room.position, logEpoch: room.log_epoch });
+                }
+            });
+        },
+        published(locator, at) {
+            const authority = [...locators].find(([, held]) => held.includes(locator))?.[0];
+            const reached = position(at.position);
+            if (authority === undefined || reached === null)
+                return;
+            void forget([...kept.values()]
+                .filter((carry) => carry.authority === authority &&
+                carry.fenced !== null &&
+                carry.fenced.logEpoch === at.logEpoch &&
+                BigInt(carry.fenced.position) <= reached)
+                .map((carry) => carry.carry));
+        },
+        reopen() {
+            const current = opening;
+            opening = null;
+            current?.close();
+        },
         track(entries) {
             locators.clear();
-            schemas.clear();
             for (const entry of entries) {
                 locators.set(entry.authority, [...(locators.get(entry.authority) ?? []), entry.locator]);
-                schemas.set(entry.authority, entry.schemas);
                 if (lines.get(entry.authority)?.logEpoch !== entry.logEpoch) {
                     lines.set(entry.authority, { logEpoch: entry.logEpoch, frames: [], held: null });
                 }
@@ -183,21 +515,22 @@ export function houseNotifications(transport, mode) {
                 if (!locators.has(authority))
                     lines.delete(authority);
             }
+            void loaded.then(() => forget([...kept.values()]
+                .filter((carry) => !locators.has(carry.authority))
+                .map((carry) => carry.carry)));
         },
-        select() { },
-        observe(notify) {
-            observers.add(notify);
-            listen();
-            return () => {
-                observers.delete(notify);
-            };
-        },
-        hold(authority, at) {
+        reach(authority, at) {
             const line = lines.get(authority);
             const held = position(at.position);
             if (line === undefined || held === null || line.logEpoch !== at.logEpoch)
                 return;
-            lines.set(authority, { ...line, held });
+            if (line.held !== null && held <= line.held)
+                return;
+            lines.set(authority, {
+                ...line,
+                frames: line.frames.filter((frame) => frame.position > held),
+                held,
+            });
         },
         advance(tracked, from, holds) {
             const line = lines.get(tracked.authority);
@@ -211,12 +544,15 @@ export function houseNotifications(transport, mode) {
             const documents = {};
             const revisions = {};
             const changes = {};
+            const binaryFiles = {};
             const removals = [];
             const applied = [];
+            const schemas = new Map();
             const forget = (path) => {
                 delete documents[path];
                 delete revisions[path];
                 delete changes[path];
+                delete binaryFiles[path];
                 const removed = removals.indexOf(path);
                 if (removed !== -1)
                     removals.splice(removed, 1);
@@ -224,19 +560,21 @@ export function houseNotifications(transport, mode) {
             for (const frame of pending) {
                 for (const entry of frame.entries) {
                     if (entry.kind === 'edited' &&
-                        entry.text === undefined &&
                         entry.revision !== undefined &&
                         holds?.(entry.path, entry.revision) === true) {
                         forget(entry.path);
                         continue;
                     }
                     applied.push(entry);
+                    if (tracked.schemas.includes(entry.path))
+                        schemas.set(entry.path, entry);
                     if (entry.kind === 'removed') {
                         forget(entry.path);
                         removals.push(entry.path);
                         continue;
                     }
-                    if (entry.text === undefined)
+                    const binary = binaryFileOf(entry);
+                    if (binary === null || (binary === undefined && entry.text === undefined))
                         return null;
                     if (mode === 'credential' && entry.revision === undefined)
                         return null;
@@ -244,9 +582,13 @@ export function houseNotifications(transport, mode) {
                     if (head === null)
                         return null;
                     forget(entry.path);
-                    documents[entry.path] = entry.text;
-                    if (entry.revision !== undefined)
-                        revisions[entry.path] = entry.revision;
+                    if (binary === undefined)
+                        documents[entry.path] = entry.text;
+                    else
+                        binaryFiles[entry.path] = binary;
+                    const revision = entry.revision ?? binary?.sha256;
+                    if (revision !== undefined)
+                        revisions[entry.path] = revision;
                     if (head !== undefined)
                         changes[entry.path] = head;
                 }
@@ -259,11 +601,13 @@ export function houseNotifications(transport, mode) {
                     roomVersion: roomVersion({ position: last.position.toString(), logEpoch: line.logEpoch }),
                     documents,
                     revisions,
+                    ...identities(revisions),
                     changes,
-                    binaryTargets: [],
+                    binaryFiles,
                     removals: removals.sort(),
                 },
                 entries: applied,
+                schemas,
             };
         },
     };