keri-ts 1.0.0 → 2.0.0

package/esm/keri/npm/src/app/mailbox/mailbox-poller.js

@@ -0,0 +1,305 @@
+/** Recipient mailbox polling: signed queries, awaited SSE intake and persisted cursor progression. */
+import { call, scoped, spawn } from "effection";
+import { abortable } from "../runtime/abortable.js";
+import { runtimeTurn } from "../runtime/runtime-turn.js";
+import { isLegacyCesrMediaRejection, readResponseDetail } from "../transport/cesr-egress.js";
+import { buildCesrRequest, CESR_CONTENT_TYPE, KERIPY_CESR_JSON_CONTENT_TYPE } from "../transport/cesr-http.js";
+import { requestWithHeaderTimeout } from "../transport/httping.js";
+import { consumeMailboxSse } from "./mailbox-sse.js";
+import { mailboxPollEndpoints } from "./mailboxing.js";
+/**
+ * Retrieve recipient messages from remote mailbox or witness endpoints over HTTP/SSE.
+ *
+ * Each endpoint has an independent worker and durable progress in the director.
+ * Retrieved records are lent to the awaited consumer; protocol intake stays outside.
+ * A provider store in the same runtime is never scanned for local recipient input.
+ */
+export class MailboxPoller {
+    /**
+     * Bind polling inputs and dependencies without starting workers or acquiring stores.
+     *
+     * @param hby Borrowed local identities and accepted-state stores.
+     * @param mailboxDirector Borrowed provider/topic/cursor owner.
+     * @param http Borrowed HTTP implementation selected by composition.
+     * @param clock Borrowed clock with epoch-millisecond now and millisecond timer delays.
+     * @param options Optional timeout overrides; finite positive values are floored with a 1 ms minimum. Missing/nonpositive/nonfinite values use 5-second headers and 30-second poll windows.
+     */
+    constructor(hby, mailboxDirector, http, clock, options = {}) {
+        /** Borrowed habitat collection used to select mailbox signers. */
+        Object.defineProperty(this, "hby", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Shared provider topics and persisted remote cursors. */
+        Object.defineProperty(this, "mailboxDirector", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Existing request-open and SSE-window settings, normalized once. */
+        Object.defineProperty(this, "timeoutPolicy", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Borrowed HTTP client passed to each polling attempt. */
+        Object.defineProperty(this, "http", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Borrowed clock for this owner's actual network deadlines. */
+        Object.defineProperty(this, "clock", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { timeouts } = options;
+        this.hby = hby;
+        this.mailboxDirector = mailboxDirector;
+        this.timeoutPolicy = normalizeMailboxPollingTimeoutPolicy(timeouts);
+        this.http = http;
+        this.clock = clock;
+    }
+    /**
+     * Register one mailbox topic that this poller should retrieve.
+     *
+     * @param topic Protocol mailbox topic; leading slash is normalized where required.
+     */
+    registerTopic(topic) {
+        this.mailboxDirector.registerTopic(topic);
+    }
+    /**
+     * Poll forever, streaming typed batches into one sink.
+     *
+     * Each remote endpoint progresses independently, and awaits the consumer before
+     * reading the next event or advancing its persisted cursor.
+     *
+     * @param onBatch Awaited endpoint-batch consumer; the durable event cursor advances only after it succeeds. Failure or cancellation leaves that event available for later retrieval.
+     * @throws Consumer/transport failures propagate; endpoint workers are joined on scope exit.
+     */
+    *pollDo(onBatch) {
+        const remoteWorkers = new Map();
+        try {
+            while (true) {
+                const topics = this.mailboxDirector.registeredTopics();
+                yield* this.syncRemoteWorkers(remoteWorkers, onBatch, topics);
+                yield* runtimeTurn();
+            }
+        }
+        finally {
+            for (const task of [...remoteWorkers.values()].reverse()) {
+                yield* task.halt();
+            }
+        }
+    }
+    /**
+     * Stream one remote response through its consumer before recording cursor progress.
+     *
+     * @param target Mailbox subject habitat plus the actual local signer (a member for groups).
+     * @param endpoint Selected remote endpoint AID and HTTP(S) URL.
+     * @param onBatch Awaited consumer; successful completion precedes durable cursor advancement.
+     */
+    *pollRemoteEndpointOnce(target, endpoint, onBatch) {
+        const cursor = this.mailboxDirector.remoteQueryCursor(target.targetPre, endpoint.eid);
+        // deno-lint-ignore no-this-alias -- Generator callbacks have no lexical this.
+        const self = this;
+        // The concrete HTTP/SSE operation owns and joins this whole poll attempt.
+        yield* call(() => fetchMailboxMessages(target.signerHab, target.targetPre, endpoint.eid, endpoint.url, cursor, mailboxFetchTimeoutPolicy(this.timeoutPolicy), this.http, this.clock, function* deliverThenAdvanceCursor(message) {
+            yield* call(() => onBatch({ pre: target.targetPre, eid: endpoint.eid, messages: [message.msg] }));
+            self.mailboxDirector.updateRemoteCursor(target.targetPre, endpoint.eid, message.topic, message.idx);
+        }));
+    }
+    /**
+     * Keep one long-lived worker running for each currently configured remote endpoint.
+     *
+     * @param remoteWorkers Owned worker map, mutated to match current endpoint selection.
+     * @param onBatch Shared awaited consumer used independently by each endpoint worker.
+     * @param topics Current registered topics; no topics means no remote workers.
+     */
+    *syncRemoteWorkers(remoteWorkers, onBatch, topics) {
+        const active = new Set();
+        if (topics.length > 0) {
+            for (const target of this.remotePollTargets()) {
+                for (const endpoint of mailboxPollEndpoints(target.hab)) {
+                    const workerKey = `${target.targetPre}:${endpoint.key}`;
+                    active.add(workerKey);
+                    if (remoteWorkers.has(workerKey)) {
+                        continue;
+                    }
+                    const task = yield* spawn(() => this.remoteEndpointWorker(target, endpoint, onBatch));
+                    remoteWorkers.set(workerKey, task);
+                }
+            }
+        }
+        for (const [workerKey, task] of [...remoteWorkers.entries()]) {
+            if (active.has(workerKey)) {
+                continue;
+            }
+            yield* task.halt();
+            remoteWorkers.delete(workerKey);
+        }
+    }
+    /**
+     * Poll one remote endpoint continuously in the long-lived runtime shape.
+     *
+     * This keeps KERIpy's "one poller per endpoint" behavior while preserving
+     * `keri-ts`'s split between mailbox transport and runtime parser ownership.
+     *
+     * @param target Mailbox subject habitat plus the actual local signer (a member for groups).
+     * @param endpoint Selected remote endpoint AID and HTTP(S) URL.
+     * @param onBatch Awaited consumer; successful completion precedes durable cursor advancement.
+     */
+    *remoteEndpointWorker(target, endpoint, onBatch) {
+        while (true) {
+            yield* this.pollRemoteEndpointOnce(target, endpoint, onBatch);
+            yield* runtimeTurn();
+        }
+    }
+    /**
+     * Select locally pollable subjects without substituting the group for its member signer.
+     *
+     * @returns Local subject/signing pairs; groups without an available local member are omitted.
+     */
+    remotePollTargets() {
+        const targets = [];
+        for (const hab of this.hby.habs.values()) {
+            const signerHab = this.pollSignerHab(hab);
+            if (!signerHab) {
+                continue;
+            }
+            targets.push({ hab, targetPre: hab.pre, signerHab });
+        }
+        return targets;
+    }
+    /**
+     * Choose the actual local signer for a subject's mailbox query.
+     *
+     * @param hab Local subject habitat, possibly a group.
+     * @returns The habitat itself, or its available local group member; null when unresolved.
+     */
+    pollSignerHab(hab) {
+        if (!this.hby.db.groups.has(hab.pre)) {
+            return hab;
+        }
+        const mid = this.hby.db.getHab(hab.pre)?.mid;
+        if (!mid) {
+            return null;
+        }
+        return this.hby.habs.get(mid) ?? null;
+    }
+}
+/**
+ * Default header acquisition (5 seconds) and absolute poll window (30 seconds).
+ */
+Object.defineProperty(MailboxPoller, "DefaultTimeoutPolicy", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: Object.freeze({
+        requestOpenTimeoutMs: 5_000,
+        maxPollDurationMs: 30_000,
+    })
+});
+/**
+ * Idle read window after the first complete event, in milliseconds.
+ */
+Object.defineProperty(MailboxPoller, "ReadIdleTimeoutMs", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 500
+});
+/**
+ * Query one endpoint and consume complete SSE events within one request scope.
+ * Header timeout or non-success status ends this attempt without events. Successful
+ * body processing and its cleanup finish before the endpoint worker begins another poll.
+ * The sole alternate submission is the exact pre-intake KERIpy media refusal:
+ * drain/close that request, then reuse the same signed query and attachments with
+ * legacy metadata. Authentication or protocol refusals do not take that branch.
+ * consume runs inline and is awaited before the caller persists its cursor; this
+ * callback is a scoped event-consumer boundary, not deferred workflow scheduling.
+ *
+ * @param hab Local signing habitat; its state/store lifetime remains with the caller.
+ * @param targetPre Mailbox subject AID; may be a group distinct from hab.
+ * @param src Remote mailbox attester AID.
+ * @param url Selected HTTP(S) polling endpoint.
+ * @param topics Next-wanted ordinals by topic.
+ * @param timeouts Header, absolute read and idle windows in milliseconds.
+ * @param http Borrowed HTTP implementation selected by composition.
+ * @param clock Borrowed clock with epoch-millisecond now and millisecond timer delays.
+ * @param consume Awaited complete-event consumer; false stops the poll and errors propagate before remote cursor advancement.
+ */
+function* fetchMailboxMessages(hab, targetPre, src, url, topics, timeouts, http, clock, consume) {
+    const query = hab.query(targetPre, src, { route: "mbx", q: { topics } });
+    const prepared = buildCesrRequest(query, { destination: src });
+    for (const contentType of [CESR_CONTENT_TYPE, KERIPY_CESR_JSON_CONTENT_TYPE]) {
+        const alternate = yield* scoped(function* () {
+            const controller = new AbortController();
+            const request = {
+                method: "POST",
+                headers: { ...prepared.headers, "Content-Type": contentType },
+                body: prepared.body,
+                signal: controller.signal,
+            };
+            const response = yield* requestWithHeaderTimeout(http, clock, url, request, timeouts.requestOpenTimeoutMs);
+            if (!response)
+                return false;
+            if (!response.ok) {
+                const detail = yield* abortable(signal => readResponseDetail(response, signal));
+                return isLegacyCesrMediaRejection(request, response.status, detail);
+            }
+            yield* consumeMailboxSse(response, controller, consume, clock, {
+                idleTimeoutMs: timeouts.readIdleTimeoutMs,
+                maxDurationMs: timeouts.maxReadDurationMs,
+            });
+            return false;
+        });
+        // Closing the refused request is part of the scoped operation above. A close
+        // failure escapes before another request can submit these same signed bytes.
+        if (!alternate)
+            return;
+    }
+}
+/**
+ * Normalize the caller's polling policy against the existing defaults.
+ *
+ * @param overrides Optional timeout values; missing/nonpositive/nonfinite values use defaults.
+ * @returns Frozen header/poll settings with finite positive inputs floored and clamped to at least 1 ms.
+ */
+function normalizeMailboxPollingTimeoutPolicy(overrides) {
+    const defaults = MailboxPoller.DefaultTimeoutPolicy;
+    return Object.freeze({
+        requestOpenTimeoutMs: positiveTimeoutMs(overrides?.requestOpenTimeoutMs, defaults.requestOpenTimeoutMs),
+        maxPollDurationMs: positiveTimeoutMs(overrides?.maxPollDurationMs, defaults.maxPollDurationMs),
+    });
+}
+/**
+ * Translate poller windows into the concrete request/SSE settings.
+ *
+ * @param timeouts Normalized poller policy.
+ * @returns Per-attempt windows with the fixed 500 ms read-idle interval.
+ */
+function mailboxFetchTimeoutPolicy(timeouts) {
+    return {
+        requestOpenTimeoutMs: timeouts.requestOpenTimeoutMs,
+        maxReadDurationMs: timeouts.maxPollDurationMs,
+        readIdleTimeoutMs: MailboxPoller.ReadIdleTimeoutMs,
+    };
+}
+/**
+ * Normalize a polling duration without turning a positive override into a zero-length window.
+ *
+ * @param value Optional proposed duration in milliseconds.
+ * @param fallback Default used for missing, nonfinite or nonpositive input.
+ * @returns Finite positive input floored with a 1 ms minimum; otherwise fallback.
+ */
+function positiveTimeoutMs(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.max(1, Math.floor(value)) : fallback;
+}

package/esm/keri/npm/src/app/mailbox/mailbox-sse.js

@@ -0,0 +1,254 @@
+import { call, resource } from "effection";
+import { abortable, cleanupFailures } from "../runtime/abortable.js";
+/** Shared encoder for complete mailbox event payloads. */
+const textEncoder = new TextEncoder();
+/**
+ * Decode one complete SSE block; an unterminated block is never delivered.
+ *
+ * @param lines Lines of one blank-line-terminated SSE block, without line endings.
+ * @returns Decoded ordinal/topic/message, or undefined for incomplete/invalid metadata.
+ */
+function eventMessage(lines) {
+    let idx = -1;
+    let topic = "";
+    const data = [];
+    for (const line of lines) {
+        const colon = line.indexOf(":");
+        const field = colon < 0 ? line : line.slice(0, colon);
+        let value = colon < 0 ? "" : line.slice(colon + 1);
+        // SSE removes exactly one optional space, not meaningful payload whitespace.
+        if (value.startsWith(" "))
+            value = value.slice(1);
+        if (field === "id")
+            idx = /^\d+$/.test(value) ? Number(value) : -1;
+        else if (field === "event")
+            topic = value;
+        else if (field === "data")
+            data.push(value);
+    }
+    if (!Number.isSafeInteger(idx) || idx < 0 || !topic || !data.length)
+        return;
+    return { idx, topic, msg: textEncoder.encode(data.join("\n")) };
+}
+/**
+ * Geometric byte growth keeps fragmentation from creating one retained metadata entry per chunk.
+ *
+ * @param bytes Current owned backing buffer.
+ * @param size Number of populated bytes already retained.
+ * @param chunk Next fragment to append.
+ * @returns Original or grown buffer containing the appended bytes.
+ */
+function appendBytes(bytes, size, chunk) {
+    if (size + chunk.length > bytes.length) {
+        const grown = new Uint8Array(Math.max(size + chunk.length, 1024, bytes.length * 2));
+        grown.set(bytes.subarray(0, size));
+        bytes = grown;
+    }
+    bytes.set(chunk, size);
+    return bytes;
+}
+/**
+ * Consume complete mailbox events with backpressure from the awaited consumer.
+ * Returning false stops the poll without acknowledging an unread event. Timeout,
+ * EOF, and cancellation discard an incomplete final event so its cursor can retry.
+ * The consumer is immediately awaited in this read owner, not scheduled or queued.
+ * Keeping consumer failure inside this scope lets cleanup retain both failures;
+ * a borrowed pull iterator alone cannot observe a later consumer exception.
+ *
+ * @param response Borrowed streaming HTTP response; caller keeps its scope open through this operation.
+ * @param controller Body cancellation owner aborted when the poll window ends.
+ * @param consume Awaited complete-event consumer; false stops the poll and errors propagate before remote cursor advancement.
+ * @param clock Borrowed clock with epoch-millisecond now and millisecond timer delays.
+ * @param options Positive idle/absolute windows in milliseconds and optional upstream signal; defaults 500 and 30000.
+ * @throws External cancellation and genuine read/consumer/cleanup errors remain visible; owned window expiry ends the poll.
+ */
+export function consumeMailboxSse(response, controller, consume, clock, options = {}) {
+    return call(function* () {
+        let lines = [];
+        let lineBytes = new Uint8Array(0);
+        let lineSize = 0;
+        const decoder = new TextDecoder();
+        let completed = false;
+        yield* consumeChunks(response, controller, function* consumeCompleteEvents(chunk) {
+            let start = 0;
+            for (let index = 0; index < chunk.length; index++) {
+                if (chunk[index] !== 10)
+                    continue;
+                lineBytes = appendBytes(lineBytes, lineSize, chunk.subarray(start, index));
+                lineSize += index - start;
+                let line = decoder.decode(lineBytes.subarray(0, lineSize));
+                if (line.endsWith("\r"))
+                    line = line.slice(0, -1);
+                lineSize = 0;
+                start = index + 1;
+                if (line) {
+                    lines.push(line);
+                }
+                else {
+                    const message = eventMessage(lines);
+                    lines = [];
+                    if (message) {
+                        completed = true;
+                        if ((yield* consume(message)) === false)
+                            return false;
+                    }
+                }
+            }
+            if (start < chunk.length) {
+                lineBytes = appendBytes(lineBytes, lineSize, chunk.subarray(start));
+                lineSize += chunk.length - start;
+            }
+        }, clock, options, () => completed);
+    });
+}
+/**
+ * Collect a raw SSE diagnostic report for one poll window. Production polling
+ * uses consumeMailboxSse to deliver complete events as they arrive.
+ *
+ * @param response Borrowed streaming HTTP response; caller keeps its scope open through this operation.
+ * @param controller Body cancellation owner aborted when the poll window ends.
+ * @param clock Borrowed clock with epoch-millisecond now and millisecond timer delays.
+ * @param options Positive idle/absolute windows in milliseconds and optional upstream signal; defaults 500 and 30000.
+ * @returns Collected text from this bounded diagnostic window.
+ * @throws External cancellation and genuine read/consumer/cleanup errors remain visible; owned window expiry ends the poll.
+ */
+export function readMailboxSseBody(response, controller, clock, options = {}) {
+    return call(function* () {
+        const decoder = new TextDecoder();
+        let bytes = new Uint8Array(0);
+        let size = 0;
+        let blockStart = 0;
+        let completed = false;
+        yield* consumeChunks(response, controller, function* collectDiagnosticChunk(chunk) {
+            bytes = appendBytes(bytes, size, chunk);
+            const end = size + chunk.length;
+            // Inspect each terminated block once. A retry/comment header alone must not start the idle deadline.
+            for (let index = size; !completed && index < end; index++) {
+                if (bytes[index] !== 10)
+                    continue;
+                if (bytes[index - 1] === 10
+                    || (bytes[index - 1] === 13 && bytes[index - 2] === 10 && bytes[index - 3] === 13)) {
+                    completed = parseMailboxSse(decoder.decode(bytes.subarray(blockStart, index + 1))).length > 0;
+                    blockStart = index + 1;
+                }
+            }
+            size = end;
+        }, clock, options, () => completed);
+        return new TextDecoder().decode(bytes.subarray(0, size));
+    });
+}
+/**
+ * Parse only terminated events from an already finite diagnostic report.
+ *
+ * @param text Finite diagnostic response text; an unterminated final block is ignored.
+ * @returns Complete valid mailbox events in received order.
+ */
+export function parseMailboxSse(text) {
+    return text.replaceAll("\r\n", "\n").split("\n\n").slice(0, -1)
+        .map((block) => eventMessage(block.split("\n")))
+        .filter((message) => message !== undefined);
+}
+/**
+ * Apply SSE deadlines to the owned byte source; the HTTP client retains native reader mechanics.
+ *
+ * @param response Borrowed streaming HTTP response; caller keeps its scope open through this operation.
+ * @param controller Body cancellation owner aborted when the poll window ends.
+ * @param consume Awaited byte-chunk consumer; false ends this window.
+ * @param clock Borrowed clock with epoch-millisecond now and millisecond timer delays.
+ * @param options Positive idle/absolute windows in milliseconds and optional upstream signal; defaults 500 and 30000.
+ * @param hasEvent Reports whether a complete event has arrived, enabling the shorter idle deadline.
+ * @throws External cancellation and genuine read/consumer/cleanup errors remain visible; owned window expiry ends the poll.
+ */
+function* consumeChunks(response, controller, consume, clock, options, hasEvent) {
+    const { idleTimeoutMs = 500, maxDurationMs = 30_000 } = options;
+    for (const value of [idleTimeoutMs, maxDurationMs]) {
+        if (!Number.isSafeInteger(value) || value < 1)
+            throw new RangeError("Invalid mailbox SSE timeout");
+    }
+    const iterator = response.body(controller.signal)[Symbol.asyncIterator]();
+    let pending;
+    let reported;
+    // Deadline cancellation wakes a blocked source read before teardown joins its iterator.
+    yield* resource(function* (provide) {
+        try {
+            yield* provide();
+        }
+        finally {
+            controller.abort();
+            const errors = [];
+            yield* call(async () => {
+                /**
+                 * Preserve cleanup failures other than the owned abort and already reported read failure.
+                 * @param error Rejection from the pending read or iterator return.
+                 */
+                const remember = (error) => {
+                    for (const remaining of cleanupFailures(error, controller.signal)) {
+                        if (!(reported && remaining === reported.error) && !errors.includes(remaining))
+                            errors.push(remaining);
+                    }
+                };
+                try {
+                    await pending;
+                }
+                catch (error) {
+                    remember(error);
+                }
+                try {
+                    await iterator.return?.();
+                }
+                catch (error) {
+                    remember(error);
+                }
+            });
+            if (errors.length && reported)
+                errors.unshift(reported.error);
+            // deno-lint-ignore no-unsafe-finally -- Keep actual source cleanup failure visible during cancellation.
+            if (errors.length === 1)
+                throw errors[0];
+            // deno-lint-ignore no-unsafe-finally -- Preserve the source failure alongside distinct cleanup failures.
+            if (errors.length)
+                throw new AggregateError(errors, "Mailbox SSE source and cleanup failed");
+        }
+    });
+    const deadline = clock.now() + maxDurationMs;
+    try {
+        while (clock.now() < deadline) {
+            const wait = Math.max(1, Math.min(hasEvent() ? idleTimeoutMs : maxDurationMs, deadline - clock.now()));
+            const next = yield* abortable(async (signal) => {
+                // This is application window policy, not native HTTP failure classification.
+                /** Wake the native body read before this child can be joined during cancellation. */
+                const cancel = () => controller.abort(signal.reason);
+                signal.addEventListener("abort", cancel, { once: true });
+                if (signal.aborted)
+                    cancel();
+                let timer;
+                pending = iterator.next();
+                try {
+                    const result = await Promise.race([
+                        pending,
+                        new Promise((resolve) => {
+                            timer = clock.setTimeout(() => resolve(undefined), wait);
+                        }),
+                    ]);
+                    if (!result)
+                        controller.abort();
+                    return result;
+                }
+                finally {
+                    if (timer !== undefined)
+                        clock.clearTimeout(timer);
+                    signal.removeEventListener("abort", cancel);
+                    await pending.catch(() => undefined);
+                }
+            }, options.signal ? AbortSignal.any([controller.signal, options.signal]) : controller.signal);
+            if (!next || next.done)
+                return;
+            if (next.value.length && (yield* consume(next.value)) === false)
+                return;
+        }
+    }
+    catch (error) {
+        reported = { error };
+        throw error;
+    }
+}