@gryt/core 0.14.0 → 0.16.0

package/dist/pairing/historySender.js

@@ -0,0 +1,182 @@
+import { generateHistoryKey, planHistoryChunks, sealHistoryChunk, } from "@gryt/crypto";
+import { archiveByDay } from "./historySnapshot.js";
+import { PairingRelayError } from "./relayClient.js";
+/** The relay's 256 MiB per session, less room for the tail. */
+export const HISTORY_SNAPSHOT_MAX_BYTES = 248 * 1024 * 1024;
+/** Entries in the envelope's manifest and in each later message, which keep under 256 and 64 KiB. */
+export const HISTORY_ENVELOPE_ENTRIES = 400;
+export const HISTORY_PAGE_ENTRIES = 180;
+/** The relay allows 64 messages a session, both sides together; N and the envelope use five. */
+const HISTORY_MAX_CHUNKS = HISTORY_ENVELOPE_ENTRIES + 40 * HISTORY_PAGE_ENTRIES;
+const MAX_NETWORK_RETRIES = 5;
+export const emptyProgress = () => ({
+    messages: 0, total: null, chunks: 0, listed: 0, missing: 0, refused: 0, truncated: false, oldest: null, complete: false,
+});
+/** A: seals the archive into chunks newest first, uploads each, then lists it to N. */
+export function createHistorySender(options) {
+    const { relay, id, token, clock, signal } = options;
+    const key = generateHistoryKey();
+    const maxBytes = options.maxBytes ?? HISTORY_SNAPSHOT_MAX_BYTES;
+    const progress = emptyProgress();
+    /** Uploaded, not yet listed. */
+    const unlisted = [];
+    let slot = 0;
+    let bytes = 0;
+    let paging = false;
+    let walked = false;
+    let posting = Promise.resolve();
+    let finished = false;
+    const report = () => options.onProgress({ ...progress });
+    async function put(n, sealed) {
+        for (let failures = 0;;) {
+            try {
+                return await relay.putChunk(id, token, n, sealed, signal);
+            }
+            catch (e) {
+                if (!(e instanceof PairingRelayError && e.code === "network") || ++failures > MAX_NETWORK_RETRIES)
+                    throw e;
+                await clock.sleep(2000, signal);
+            }
+        }
+    }
+    /** False once the cap or the relay says no more. */
+    async function upload(records, capped) {
+        for (const chunk of planHistoryChunks(records)) {
+            if (signal.aborted)
+                throw new Error("History transfer cancelled.");
+            const { entry, sealed } = sealHistoryChunk(key, chunk);
+            if (capped && (bytes + sealed.length > maxBytes || slot >= HISTORY_MAX_CHUNKS))
+                return false;
+            const n = slot++;
+            try {
+                await put(n, sealed);
+            }
+            catch (e) {
+                // The relay's own caps: whatever didn't fit stays behind, the same as our own cap.
+                if (capped && e instanceof PairingRelayError && (e.status === 507 || e.code === "busy"))
+                    return false;
+                throw e;
+            }
+            bytes += sealed.length;
+            unlisted.push({ n, ...entry });
+            progress.chunks++;
+            progress.messages += entry.count;
+            progress.oldest = Math.min(progress.oldest ?? entry.first, entry.first);
+            report();
+        }
+        return true;
+    }
+    async function walk() {
+        const conversations = await options.archive.conversations();
+        if (conversations.every((c) => typeof c.count === "number")) {
+            progress.total = conversations.reduce((sum, c) => sum + (c.count ?? 0), 0);
+            report();
+        }
+        for await (const day of archiveByDay(options.archive)) {
+            if (!(await upload(day, true))) {
+                progress.truncated = true;
+                break;
+            }
+            if (paging)
+                void flush(false, false);
+        }
+        walked = true;
+        if (paging)
+            void flush(false);
+    }
+    const snapshot = walk();
+    snapshot.catch(() => undefined);
+    /** Lists what's uploaded, a page per message. `partial` false keeps back a page that isn't full. */
+    function flush(final, partial = true) {
+        const pages = [];
+        while (unlisted.length >= (partial ? 1 : HISTORY_PAGE_ENTRIES))
+            pages.push(unlisted.splice(0, HISTORY_PAGE_ENTRIES));
+        if (final && pages.length === 0)
+            pages.push([]);
+        pages.forEach((chunks, i) => {
+            const message = { t: "history", chunks };
+            if (progress.total !== null)
+                message.total = progress.total;
+            if (progress.truncated)
+                message.truncated = true;
+            if (final && i === pages.length - 1)
+                message.final = true;
+            posting = posting.then(async () => {
+                await options.post(message);
+                progress.listed += chunks.length;
+                if (message.final)
+                    progress.complete = true;
+                report();
+            });
+        });
+        posting.catch(() => undefined);
+        return posting;
+    }
+    return {
+        key,
+        progress: () => ({ ...progress }),
+        /** Waits until `until` or the end of the snapshot, whichever is first, and hands over what's up. */
+        async forEnvelope(until) {
+            const wait = new AbortController();
+            const stop = () => wait.abort();
+            signal.addEventListener("abort", stop, { once: true });
+            try {
+                await Promise.race([snapshot, clock.sleep(Math.max(0, until - clock.now()), wait.signal).catch(() => undefined)]);
+            }
+            finally {
+                wait.abort();
+                signal.removeEventListener("abort", stop);
+            }
+            const listed = unlisted.splice(0, HISTORY_ENVELOPE_ENTRIES);
+            progress.listed += listed.length;
+            report();
+            return listed.map(({ n: _n, ...entry }) => entry);
+        },
+        /** After N's "ready": list what's waiting, and keep listing as pages fill. */
+        startPaging() {
+            paging = true;
+            void flush(false, walked);
+        },
+        /** The tail: records past the snapshot. `final` waits for the snapshot and says that's all. */
+        async send(records, final) {
+            if (finished)
+                throw new Error("This history transfer has finished.");
+            if (final) {
+                await snapshot;
+                finished = true;
+            }
+            if (records.length > 0)
+                await upload(records, false);
+            await flush(final);
+        },
+    };
+}
+/**
+ * A's record of MLS messages it archived during the transfer. The tail is every one past the
+ * snapshot that N can't read: before N's add, or sent in an older epoch that landed after it.
+ */
+export function createTailRecorder() {
+    let on = false;
+    let noted = [];
+    return {
+        start: () => void (on = true),
+        note(message) {
+            if (on)
+                noted.push(message);
+        },
+        /** Everything noted since the last call that N needs. `snap` and `added` are per host. */
+        take(snap, added) {
+            const out = [];
+            for (const m of noted) {
+                if (m.seq <= (snap.get(m.host)?.get(m.conversationId) ?? -1))
+                    continue;
+                const add = added[m.host]?.find((a) => a.conversationId === m.conversationId)?.add ?? null;
+                if (add && m.seq >= add.seq && m.epoch >= add.epoch)
+                    continue;
+                out.push(m.record);
+            }
+            noted = [];
+            return out;
+        },
+    };
+}

package/dist/pairing/historySnapshot.d.ts

@@ -0,0 +1,7 @@
+import type { HistoryRecord } from "@gryt/crypto";
+import type { HistoryArchive, HistoryCursor } from "./interfaces.js";
+export declare const dayOf: (sentAt: number) => number;
+/** Negative when `a` is newer: by time, then by message id, the same order the archive pages in. */
+export declare function newerFirst(a: HistoryCursor, b: HistoryCursor): number;
+/** A's archive one UTC day at a time, newest day first across every conversation and server. */
+export declare function archiveByDay(archive: HistoryArchive, pageSize?: number): AsyncGenerator<HistoryRecord[]>;

package/dist/pairing/historySnapshot.js

@@ -0,0 +1,45 @@
+const DAY_MS = 24 * 60 * 60 * 1000;
+export const dayOf = (sentAt) => Math.floor(sentAt / DAY_MS);
+/** Negative when `a` is newer: by time, then by message id, the same order the archive pages in. */
+export function newerFirst(a, b) {
+    return b.sentAt - a.sentAt || (a.messageId < b.messageId ? 1 : a.messageId > b.messageId ? -1 : 0);
+}
+/** A's archive one UTC day at a time, newest day first across every conversation and server. */
+export async function* archiveByDay(archive, pageSize = 500) {
+    const walks = (await archive.conversations()).map((c) => ({ ...c, buffer: [], exhausted: false }));
+    const fill = async (w) => {
+        if (w.buffer.length > 0 || w.exhausted)
+            return;
+        const page = await archive.page(w.scope, w.conversationId, { before: w.before, limit: pageSize });
+        // Only what's older than the cursor and in this conversation, so a sloppy page can't loop forever.
+        const fresh = page
+            .filter((r) => r.scope === w.scope && r.conversationId === w.conversationId)
+            .filter((r) => !w.before || newerFirst(r, w.before) > 0)
+            .sort(newerFirst);
+        w.buffer = fresh;
+        w.before = fresh.at(-1) ?? w.before;
+        w.exhausted = fresh.length === 0 || page.length < pageSize;
+    };
+    for (;;) {
+        await Promise.all(walks.map(fill));
+        const heads = walks.filter((w) => w.buffer.length > 0);
+        if (heads.length === 0)
+            return;
+        const day = Math.max(...heads.map((w) => dayOf(w.buffer[0].sentAt)));
+        const out = [];
+        for (const w of heads) {
+            for (;;) {
+                let i = 0;
+                while (i < w.buffer.length && dayOf(w.buffer[i].sentAt) === day)
+                    out.push(w.buffer[i++]);
+                w.buffer = w.buffer.slice(i);
+                if (w.buffer.length > 0)
+                    break;
+                await fill(w);
+                if (w.buffer.length === 0)
+                    break;
+            }
+        }
+        yield out;
+    }
+}

package/dist/pairing/interfaces.d.ts

@@ -1,5 +1,5 @@
-import type { PairingEnvelope } from "@gryt/crypto";
-import type { MlsAddOwnDeviceOptions, MlsOwnDeviceAdd } from "../mls/interfaces.js";
+import type { HistoryRecord, PairingEnvelope } from "@gryt/crypto";
+import type { MlsAddOwnDeviceOptions, MlsGroupPosition, MlsOwnDeviceAdd } from "../mls/interfaces.js";
 /** `fetch`, cut down to what pairing uses. The browser's, Node's and React Native's all fit. */
 export type PairingFetch = (url: string, init: {
     method: string;
@@ -69,6 +69,73 @@ export interface PairedServerDevice {
 /** A's DM driver on one server, or undefined when A isn't on it. */
 export type OwnDeviceAdder = (host: string) => {
     addOwnDevice(deviceId: string, options?: MlsAddOwnDeviceOptions): Promise<MlsOwnDeviceAdd[]>;
+    /** Where the history snapshot stops in each group; the tail starts after it. */
+    groupPositions(): Promise<MlsGroupPosition[]>;
 } | undefined;
-/** Why a pairing stopped. `approve:<error>` carries the extension's own error code. */
-export type PairingEndReason = "cancelled" | "cancelled_by_other" | "mismatch" | "timed_out" | "expired" | "already_claimed" | "unknown_code" | "wrong_relay" | "newer_version" | "not_pairing" | "rate_limited" | "tampered" | "wrong_account" | "sign_in_failed" | "relay_error" | `approve:${string}`;
+/** Why a pairing stopped. `approve:<error>` carries the extension's own error code for anything
+    below that isn't broken out on its own (GRYT-1578; auth#46 has the full list). */
+export type PairingEndReason = "cancelled" | "cancelled_by_other" | "mismatch" | "timed_out" | "expired" | "already_claimed" | "unknown_code" | "wrong_relay" | "newer_version" | "not_pairing" | "rate_limited" | "tampered" | "wrong_account" | "sign_in_failed" | "relay_error"
+/** N's archive refused the history it was handed. Keys and sign-in are in place. */
+ | "history_failed"
+/** The approve endpoint says this user_code already got its one answer. */
+ | "code_used"
+/** The approve endpoint says the user_code isn't valid any more: unknown, expired, or not pending. */
+ | "code_expired"
+/** The approve endpoint refuses because the account has something pending, like verifying an email. */
+ | "required_actions"
+/** The approve endpoint refuses A's access token as too old; refresh it and try again. */
+ | "stale_token"
+/** The device grant itself said no: A denied it, or the approve endpoint's refusal denied the code. */
+ | "access_denied"
+/** The device grant's user_code ran out before anyone answered it. */
+ | "expired_token" | `approve:${string}`;
+/** A conversation in A's archive. `count` feeds the progress total, so leave it off rather than guess. */
+export interface HistoryConversation {
+    scope: string;
+    conversationId: string;
+    count?: number;
+}
+/** Where a page of the archive stops: the oldest record in it. */
+export interface HistoryCursor {
+    sentAt: number;
+    messageId: string;
+}
+/** A's archive, read for the history snapshot. Records are @gryt/crypto's HistoryRecord. */
+export interface HistoryArchive {
+    conversations(): Promise<HistoryConversation[]>;
+    /** Up to `limit` records sent before `before` (by sentAt, then messageId), in any order. */
+    page(scope: string, conversationId: string, options: {
+        before?: HistoryCursor;
+        limit: number;
+    }): Promise<HistoryRecord[]>;
+}
+/** N's archive. Core hands it each message id once; a record for an id N already holds replaces it. */
+export interface HistorySink {
+    put(records: HistoryRecord[]): Promise<void>;
+}
+/** How a history transfer is going, on either side. */
+export interface HistoryProgress {
+    /** A: messages uploaded and listed. N: messages stored. */
+    messages: number;
+    /** A: from the archive's counts. N: what A said it would send. Null when nobody knows yet. */
+    total: number | null;
+    /** Chunks uploaded (A) or fetched and stored (N), out of `listed`. */
+    chunks: number;
+    listed: number;
+    /** N only: chunks the relay no longer has, and chunks that failed their manifest check. */
+    missing: number;
+    refused: number;
+    /** The relay's cap stopped the snapshot early; the oldest messages stayed behind. */
+    truncated: boolean;
+    /** The oldest message time sent so far, so N can say how far back it has. */
+    oldest: number | null;
+    complete: boolean;
+}
+/** A message A archived from MLS, received or sent, with where it sits in its group's log. */
+export interface HistoryNotedMessage {
+    host: string;
+    conversationId: string;
+    seq: number;
+    epoch: number;
+    record: HistoryRecord;
+}

package/dist/pairing/newDevice.d.ts

@@ -1,5 +1,5 @@
 import { type PairingEmoji, type PairingServer } from "@gryt/crypto";
-import type { PairedServerDevice, PairingClock, PairingDeviceInfo, PairingEndReason, PairingOidc, PairingStorage } from "./interfaces.js";
+import type { HistoryProgress, HistorySink, PairedServerDevice, PairingClock, PairingDeviceInfo, PairingEndReason, PairingOidc, PairingStorage } from "./interfaces.js";
 import type { PairingRelay } from "./relayClient.js";
 export type NewDeviceState = {
     phase: "idle";
@@ -46,10 +46,15 @@ export interface NewDeviceOptions {
     /** Only for a server with its own auth server: goes in the QR for A to check against its own. */
     relayOrigin?: string;
     approvalMs?: number;
+    /** N's archive. Without it, history A sends is left on the relay. */
+    history?: HistorySink;
 }
 export interface NewDevicePairing {
     readonly state: NewDeviceState;
     subscribe(listener: (state: NewDeviceState) => void): () => void;
+    /** Null until the envelope brings a history key, or with no sink. */
+    readonly history: HistoryProgress | null;
+    subscribeHistory(listener: (progress: HistoryProgress) => void): () => void;
     start(): void;
     /** In `joining`: N's device on each server, once its KeyPackages are up. */
     ready(devices: PairedServerDevice[]): Promise<void>;

package/dist/pairing/newDevice.js

@@ -1,7 +1,8 @@
 import { base64Url, base64UrlDecode, createPairingKey, decodePairingEnvelope, decodePairingSessionId, formatPairingCode, formatPairingQr, pairingCommitment, startNewDeviceSession, } from "@gryt/crypto";
 import { sha256 } from "@noble/hashes/sha2.js";
 import { randomBytes } from "@noble/hashes/utils.js";
-import { mailbox, PairingEnded, reasonFor, sealJson, systemClock } from "./channel.js";
+import { mailbox, openJson, PairingEnded, reasonFor, sealJson, systemClock } from "./channel.js";
+import { createHistoryReceiver } from "./historyReceiver.js";
 export const PAIRING_APPROVAL_MS = 60_000;
 const PAIRING_OIDC_SCOPE = "openid profile email offline_access";
 /** N: shows the QR and code, checks the emoji, opens the envelope, signs in, then says ready. */
@@ -13,6 +14,14 @@ export function createNewDevicePairing(options) {
     const abort = new AbortController();
     let state = { phase: "idle" };
     let current = null;
+    let receiver = null;
+    let history = null;
+    const historyListeners = new Set();
+    const setHistory = (progress) => {
+        history = progress;
+        for (const l of historyListeners)
+            l(progress);
+    };
     const set = (next) => {
         if (state.phase === "ended" || state.phase === "done")
             return;
@@ -116,10 +125,15 @@ export function createNewDevicePairing(options) {
                     throw failure;
                 if (polled.status === "ok")
                     return checkIdToken(polled.tokens, account, session.keycloakNonce);
+                // RFC 8628's own two terminal errors, plus what auth#46 leaves behind when A's approve
+                // endpoint refuses the code: either way N's device grant sees it as denied.
+                if (polled.status === "denied")
+                    throw new PairingEnded("access_denied");
+                if (polled.status === "expired")
+                    throw new PairingEnded("expired_token");
                 if (polled.status === "slow_down")
                     interval += 5000;
-                else if (polled.status !== "pending")
-                    break;
+                // Anything else is "pending": keep polling.
             }
             throw new PairingEnded("sign_in_failed");
         }
@@ -136,6 +150,13 @@ export function createNewDevicePairing(options) {
             const envelope = outcome;
             const tokens = envelope.account ? await signIn(envelope.account, envelope.from) : null;
             await storage.commit(envelope, tokens);
+            if (envelope.history && options.history) {
+                const { id, token } = current;
+                receiver = createHistoryReceiver({
+                    relay, id, token, key: envelope.history.key, sink: options.history, clock, signal: abort.signal, onProgress: setHistory,
+                });
+                receiver.listManifest(envelope.history.manifest.chunks);
+            }
             set({ phase: "joining", servers: envelope.servers, from: envelope.from });
         }
         catch (e) {
@@ -143,20 +164,43 @@ export function createNewDevicePairing(options) {
                 await end(reasonFor(e));
         }
     }
+    /** Reads A's history messages until every chunk is in, or A closing is the finish. */
     async function linked(from) {
+        const { box, session } = current;
+        const stop = new AbortController();
+        const onAbort = () => stop.abort();
+        abort.signal.addEventListener("abort", onAbort, { once: true });
+        let failed = false;
+        receiver?.done.then(onAbort, () => ((failed = true), onAbort()));
         try {
-            // History from A lands here in a later version. Until then, A closing is the finish.
-            while (await current.box.next(abort.signal))
-                ;
+            for (;;) {
+                const body = openJson(session, await box.next(stop.signal), "history");
+                if (receiver)
+                    receiver.list(body);
+                // No sink here: nothing to fetch, so A's last message is the finish.
+                else if (body.final === true)
+                    throw (stop.abort(), new Error("finished"));
+            }
         }
         catch (e) {
             if (abort.signal.aborted)
                 return;
+            if (failed)
+                return end("history_failed");
+            if (stop.signal.aborted) {
+                set({ phase: "done", from });
+                session?.close();
+                return void (await closeQuietly());
+            }
             const reason = reasonFor(e);
             if (reason !== "cancelled_by_other" && reason !== "expired")
                 return end(reason);
+            receiver?.abandon();
             set({ phase: "done", from });
-            current?.session?.close();
+            session?.close();
+        }
+        finally {
+            abort.signal.removeEventListener("abort", onAbort);
         }
     }
     return {
@@ -167,6 +211,13 @@ export function createNewDevicePairing(options) {
             listeners.add(listener);
             return () => void listeners.delete(listener);
         },
+        get history() {
+            return history;
+        },
+        subscribeHistory(listener) {
+            historyListeners.add(listener);
+            return () => void historyListeners.delete(listener);
+        },
         start() {
             if (state.phase !== "idle")
                 throw new Error("This pairing has started already.");

package/dist/pairing/relayClient.d.ts

@@ -40,6 +40,13 @@ export interface PairingRelay {
     /** The other side's messages after `after`, waiting up to `waitSeconds`. Asking past one acks it. */
     poll(id: string, token: string, after: number, waitSeconds: number, signal?: AbortSignal): Promise<RelayMessage[]>;
     close(id: string, token: string): Promise<void>;
+    /** A: one sealed history chunk into slot `n`. */
+    putChunk(id: string, token: string, n: number, sealed: Uint8Array, signal?: AbortSignal): Promise<void>;
+    /** N: the chunk in slot `n`. A 404 means the relay doesn't have it. */
+    getChunk(id: string, token: string, n: number, signal?: AbortSignal): Promise<Uint8Array>;
+    /** N: done with slot `n`. */
+    deleteChunk(id: string, token: string, n: number): Promise<void>;
 }
-/** The relay's HTTP API as auth#45 built it, under `{origin}/api/v1/pairing`. */
+/** The relay's HTTP API as auth#45 built it, under `{origin}/api/v1/pairing`. The chunk
+    calls follow docs/pairing-design.md; the relay doesn't have them yet (GRYT-1591). */
 export declare function createPairingRelay(origin: string, fetch: PairingFetch): PairingRelay;

package/dist/pairing/relayClient.js

@@ -1,3 +1,4 @@
+import { base64Url, base64UrlDecode } from "@gryt/crypto";
 /** A refusal from the relay: its status and the `error` it sent, or "network" when nothing came back. */
 export class PairingRelayError extends Error {
     status;
@@ -9,7 +10,8 @@ export class PairingRelayError extends Error {
         this.code = code;
     }
 }
-/** The relay's HTTP API as auth#45 built it, under `{origin}/api/v1/pairing`. */
+/** The relay's HTTP API as auth#45 built it, under `{origin}/api/v1/pairing`. The chunk
+    calls follow docs/pairing-design.md; the relay doesn't have them yet (GRYT-1591). */
 export function createPairingRelay(origin, fetch) {
     const base = `${origin.replace(/\/+$/, "")}/api/v1/pairing/sessions`;
     async function call(method, path, body, token, signal) {
@@ -44,5 +46,18 @@ export function createPairingRelay(origin, fetch) {
             return (Array.isArray(res.messages) ? res.messages : []);
         },
         close: async (id, token) => void (await call("DELETE", path(id), undefined, token)),
+        putChunk: async (id, token, n, sealed, signal) => void (await call("PUT", `${path(id)}/chunks/${n}`, { body: base64Url(sealed) }, token, signal)),
+        getChunk: async (id, token, n, signal) => {
+            const res = await call("GET", `${path(id)}/chunks/${n}`, undefined, token, signal);
+            if (typeof res.body !== "string")
+                throw new PairingRelayError(200, "invalid_body");
+            try {
+                return base64UrlDecode(res.body);
+            }
+            catch {
+                throw new PairingRelayError(200, "invalid_body");
+            }
+        },
+        deleteChunk: async (id, token, n) => void (await call("DELETE", `${path(id)}/chunks/${n}`, undefined, token)),
     };
 }

package/dist/testing.d.ts

@@ -0,0 +1 @@
+export { FakeDeliveryService, MemoryMlsStore } from "./mls/deliveryService.fake.js";

package/dist/testing.js

@@ -0,0 +1,3 @@
+/* For the apps' own tests: the in-memory delivery service and state store the driver's tests
+   use, so a session can run the real driver against it. Not for app code. */
+export { FakeDeliveryService, MemoryMlsStore } from "./mls/deliveryService.fake.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.14.0",
+  "version": "0.16.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {
@@ -14,6 +14,10 @@
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
+    },
+    "./testing": {
+      "types": "./dist/testing.d.ts",
+      "default": "./dist/testing.js"
     }
   },
   "files": [
@@ -35,7 +39,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@gryt/crypto": "0.8.0",
+    "@gryt/crypto": "0.9.0",
     "@noble/hashes": "^2.3.0"
   }
 }