@gryt/core 0.32.1 → 0.33.1

package/dist/index.d.ts

@@ -22,4 +22,6 @@ export type { DmSealingMode, MlsAddOwnDeviceOptions, MlsClaimedKeyPackage, MlsDe
 export { createApproverPairing, type ApproverOptions, type ApproverPairing, type ApproverState, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, type NewDeviceOptions, type NewDevicePairing, type NewDeviceState, } from "./pairing/newDevice.js";
 export { createPairingRelay, PairingRelayError, type PairingRelay } from "./pairing/relayClient.js";
+export { chooseHistoryServer, historyStoreError, openServerHistory, serverHistoryStore, type HistoryFetch, type ServerAccess, } from "./pairing/serverHistory.js";
+export { HistoryStoreError, type HistoryChunkStore, type HistoryStoreChooser, type HistoryStoreOpener, } from "./pairing/historyStore.js";
 export type { DeviceAuthorization, DeviceTokenPoll, HistoryArchive, HistoryConversation, HistoryCursor, HistoryNotedMessage, HistoryProgress, HistorySink, OwnDeviceAdder, PairedServerDevice, PairingClock, PairingDeviceInfo, PairingEndReason, PairingFetch, PairingOidc, PairingStorage, PairingTokens, } from "./pairing/interfaces.js";

package/dist/index.js

@@ -23,3 +23,5 @@ export { importId, retryAfterMs, retryLeftOut, runDiscordImport, } from "./impor
 export { createApproverPairing, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, } from "./pairing/newDevice.js";
 export { createPairingRelay, PairingRelayError } from "./pairing/relayClient.js";
+export { chooseHistoryServer, historyStoreError, openServerHistory, serverHistoryStore, } from "./pairing/serverHistory.js";
+export { HistoryStoreError, } from "./pairing/historyStore.js";

package/dist/pairing/approver.d.ts

@@ -1,5 +1,6 @@
 import { type PairingEmoji, type PairingEnvelope } from "@gryt/crypto";
 import type { MlsOwnDeviceAdd } from "../mls/interfaces.js";
+import type { HistoryStoreChooser } from "./historyStore.js";
 import type { HistoryArchive, HistoryNotedMessage, HistoryProgress, OwnDeviceAdder, PairingClock, PairingDeviceInfo, PairingEndReason, PairingFetch } from "./interfaces.js";
 import type { PairingRelay } from "./relayClient.js";
 export type ApproverState = {
@@ -59,6 +60,8 @@ export interface ApproverOptions {
     approvalMs?: number;
     /** A's archive. Without it, no history goes across. */
     history?: HistoryArchive;
+    /** Picks the server that holds the history. Without one, the history stays on A. */
+    historyStore?: HistoryStoreChooser;
     /** How long after the adds A keeps collecting old-epoch messages for the last tail. */
     lateWindowMs?: number;
 }
@@ -67,6 +70,8 @@ export interface ApproverPairing {
     subscribe(listener: (state: ApproverState) => void): () => void;
     /** Null until Approve, or with no archive. */
     readonly history: HistoryProgress | null;
+    /** True once Approve found no server to hold the history, so it stays on A. */
+    readonly historyLeftBehind: boolean;
     subscribeHistory(listener: (progress: HistoryProgress) => void): () => void;
     /** Every MLS message A archives while this runs, received or sent, for the tail. */
     noteMessage(message: HistoryNotedMessage): void;

package/dist/pairing/approver.js

@@ -30,6 +30,7 @@ export function createApproverPairing(options) {
     let confirming = null;
     let deadline = 0;
     let history = null;
+    let historyLeftBehind = false;
     const historyListeners = new Set();
     const tail = createTailRecorder();
     const setHistory = (progress) => {
@@ -187,10 +188,10 @@ export function createApproverPairing(options) {
         }
         return snap;
     }
-    function startHistory(archive) {
+    function startHistory(archive, store) {
         const { id, token, session } = confirmed();
         return createHistorySender({
-            relay, id, token, archive, clock, signal: abort.signal,
+            store, archive, clock, signal: abort.signal,
             post: (message) => relay.post(id, token, { type: "sealed", body: sealJson(session, message) }),
             onProgress: setHistory,
         });
@@ -201,13 +202,17 @@ export function createApproverPairing(options) {
         let sender = null;
         let snap = new Map();
         let sealed = envelope;
-        if (options.history) {
+        const held = options.history && options.historyStore ? await options.historyStore(envelope.servers).catch(() => null) : null;
+        if (options.history && held) {
             // Recording starts before the positions are read, so nothing lands between the two unseen.
             tail.start();
             snap = await positions(envelope);
-            sender = startHistory(options.history);
+            sender = startHistory(options.history, held.store);
             const chunks = await sender.forEnvelope(Math.min(clock.now() + ENVELOPE_BUDGET_MS, deadline - 10_000));
-            sealed = { ...envelope, history: { key: sender.key, manifest: { v: 1, chunks } } };
+            sealed = { ...envelope, history: { key: sender.key, manifest: { v: 1, chunks }, host: held.host, transfer: held.transfer } };
+        }
+        else if (options.history && envelope.servers.length > 0) {
+            historyLeftBehind = true;
         }
         await relay.post(id, token, { type: "sealed", body: base64Url(session.seal(encodePairingEnvelope(sealed))) });
         if (envelope.account && accessToken) {
@@ -234,7 +239,10 @@ export function createApproverPairing(options) {
         }
         set({ phase: "sending", device, added });
         await sender.send(tail.take(snap, added), false);
-        await clock.sleep(options.lateWindowMs ?? HISTORY_LATE_WINDOW_MS, abort.signal);
+        // Late old-epoch messages can only come from a group N was just added to.
+        if (Object.values(added).some((groups) => groups.length > 0)) {
+            await clock.sleep(options.lateWindowMs ?? HISTORY_LATE_WINDOW_MS, abort.signal);
+        }
         await sender.send(tail.take(snap, added), true);
         // N closes the session once it has every chunk; closing here would delete them.
         set({ phase: "done", device, added });
@@ -251,6 +259,9 @@ export function createApproverPairing(options) {
         get history() {
             return history;
         },
+        get historyLeftBehind() {
+            return historyLeftBehind;
+        },
         subscribeHistory(listener) {
             historyListeners.add(listener);
             return () => void historyListeners.delete(listener);

package/dist/pairing/historyReceiver.d.ts

@@ -1,10 +1,8 @@
 import { type HistoryChunkEntry } from "@gryt/crypto";
 import type { HistoryProgress, HistorySink, PairingClock } from "./interfaces.js";
-import { type PairingRelay } from "./relayClient.ts.js";
+import type { HistoryChunkStore } from "./historyStore.js";
 export interface HistoryReceiverOptions {
-    relay: PairingRelay;
-    id: string;
-    token: string;
+    store: HistoryChunkStore;
     key: Uint8Array;
     sink: HistorySink;
     clock: PairingClock;
@@ -13,7 +11,7 @@ export interface HistoryReceiverOptions {
 }
 /** N: fetches each listed chunk newest first, checks it against its entry, and stores each message once. */
 export declare function createHistoryReceiver(options: HistoryReceiverOptions): {
-    done: Promise<void>;
+    done: Promise<undefined>;
     progress: () => {
         messages: number;
         total: number | null;

package/dist/pairing/historyReceiver.js

@@ -1,12 +1,12 @@
 import { openHistoryChunk, parseHistoryChunkEntry } from "@gryt/crypto";
 import { PairingEnded } from "./channel.js";
 import { emptyProgress } from "./historySender.js";
-import { PairingRelayError } from "./relayClient.js";
+import { HistoryStoreError } from "./historyStore.js";
 const MAX_NETWORK_RETRIES = 5;
 const messageKey = (r) => JSON.stringify([r.scope, r.conversationId, r.messageId]);
 /** N: fetches each listed chunk newest first, checks it against its entry, and stores each message once. */
 export function createHistoryReceiver(options) {
-    const { relay, id, token, key, sink, clock, signal } = options;
+    const { store, key, sink, clock, signal } = options;
     const progress = emptyProgress();
     const pending = new Map();
     const slots = new Set();
@@ -36,14 +36,12 @@ export function createHistoryReceiver(options) {
     async function fetchChunk(n) {
         for (let failures = 0;;) {
             try {
-                return await relay.getChunk(id, token, n, signal);
+                return await store.get(n, signal);
             }
             catch (e) {
-                if (e instanceof PairingRelayError && e.status === 404)
+                if (e instanceof HistoryStoreError && e.code === "missing")
                     return "missing";
-                if (e instanceof PairingRelayError && e.status === 410)
-                    return "gone";
-                if (!(e instanceof PairingRelayError && e.code === "network") || ++failures > MAX_NETWORK_RETRIES)
+                if (!(e instanceof HistoryStoreError && e.code === "network") || ++failures > MAX_NETWORK_RETRIES)
                     throw e;
                 await clock.sleep(2000, signal);
             }
@@ -77,8 +75,6 @@ export function createHistoryReceiver(options) {
             }
             const [n, entry] = item;
             const sealed = await fetchChunk(n);
-            if (sealed === "gone")
-                return abandon();
             pending.delete(n);
             if (sealed === "missing") {
                 progress.missing++;
@@ -103,7 +99,7 @@ export function createHistoryReceiver(options) {
             progress.messages += fresh.length;
             progress.oldest = Math.min(progress.oldest ?? entry.first, entry.first);
             report();
-            relay.deleteChunk(id, token, n).catch(() => undefined);
+            store.delete(n).catch(() => undefined);
         }
     }
     signal.addEventListener("abort", nudge, { once: true });

package/dist/pairing/historySender.d.ts

@@ -1,8 +1,8 @@
 import { type HistoryChunkEntry, type HistoryRecord } from "@gryt/crypto";
 import type { MlsOwnDeviceAdd } from "../mls/interfaces.js";
 import type { HistoryArchive, HistoryNotedMessage, HistoryProgress, PairingClock } from "./interfaces.js";
-import { type PairingRelay } from "./relayClient.ts.js";
-/** The relay's 256 MiB per session, less room for the tail. */
+import type { HistoryChunkStore } from "./historyStore.js";
+/** The server's 256 MiB per user, less room for the tail. */
 export declare const HISTORY_SNAPSHOT_MAX_BYTES: number;
 /** Entries in the envelope's manifest and in each later message, which keep under 256 and 64 KiB. */
 export declare const HISTORY_ENVELOPE_ENTRIES = 400;
@@ -21,9 +21,7 @@ export interface HistoryMessage {
 }
 export declare const emptyProgress: () => HistoryProgress;
 export interface HistorySenderOptions {
-    relay: PairingRelay;
-    id: string;
-    token: string;
+    store: HistoryChunkStore;
     archive: HistoryArchive;
     clock: PairingClock;
     signal: AbortSignal;

package/dist/pairing/historySender.js

@@ -1,7 +1,7 @@
 import { generateHistoryKey, planHistoryChunks, sealHistoryChunk, } from "@gryt/crypto";
 import { archiveByDay } from "./historySnapshot.js";
-import { PairingRelayError } from "./relayClient.js";
-/** The relay's 256 MiB per session, less room for the tail. */
+import { HistoryStoreError } from "./historyStore.js";
+/** The server's 256 MiB per user, less room for the tail. */
 export const HISTORY_SNAPSHOT_MAX_BYTES = 248 * 1024 * 1024;
 /** Entries in the envelope's manifest and in each later message, which keep under 256 and 64 KiB. */
 export const HISTORY_ENVELOPE_ENTRIES = 400;
@@ -14,7 +14,7 @@ export const emptyProgress = () => ({
 });
 /** A: seals the archive into chunks newest first, uploads each, then lists it to N. */
 export function createHistorySender(options) {
-    const { relay, id, token, clock, signal } = options;
+    const { store, clock, signal } = options;
     const key = generateHistoryKey();
     const maxBytes = options.maxBytes ?? HISTORY_SNAPSHOT_MAX_BYTES;
     const progress = emptyProgress();
@@ -30,16 +30,16 @@ export function createHistorySender(options) {
     async function put(n, sealed) {
         for (let failures = 0;;) {
             try {
-                return await relay.putChunk(id, token, n, sealed, signal);
+                return await store.put(n, sealed, signal);
             }
             catch (e) {
-                if (!(e instanceof PairingRelayError && e.code === "network") || ++failures > MAX_NETWORK_RETRIES)
+                if (!(e instanceof HistoryStoreError && e.code === "network") || ++failures > MAX_NETWORK_RETRIES)
                     throw e;
                 await clock.sleep(2000, signal);
             }
         }
     }
-    /** False once the cap or the relay says no more. */
+    /** False once the cap or the store says no more. */
     async function upload(records, capped) {
         for (const chunk of planHistoryChunks(records)) {
             if (signal.aborted)
@@ -52,8 +52,8 @@ export function createHistorySender(options) {
                 await put(n, sealed);
             }
             catch (e) {
-                // The relay's own caps: whatever didn't fit stays behind, the same as our own cap.
-                if (capped && e instanceof PairingRelayError && (e.status === 507 || e.code === "busy"))
+                // The server's own caps: whatever didn't fit stays behind, the same as our own cap.
+                if (capped && e instanceof HistoryStoreError && e.code === "full")
                     return false;
                 throw e;
             }

package/dist/pairing/historyStore.d.ts

@@ -0,0 +1,20 @@
+import type { PairingServer } from "@gryt/crypto";
+/** Where A leaves sealed history chunks and N picks them up: one of the user's own servers, never Gryt's relay (GRYT-2055). */
+export interface HistoryChunkStore {
+    put(n: number, sealed: Uint8Array, signal?: AbortSignal): Promise<void>;
+    get(n: number, signal?: AbortSignal): Promise<Uint8Array>;
+    delete(n: number): Promise<void>;
+}
+/** "network" is retried, "full" ends the snapshot early, "missing" is a chunk the store doesn't have. */
+export declare class HistoryStoreError extends Error {
+    readonly code: "network" | "full" | "missing" | "refused";
+    constructor(code: HistoryStoreError["code"], message?: string);
+}
+/** A's choice of a server both devices are on that will hold the history, or null when none will. */
+export type HistoryStoreChooser = (servers: PairingServer[]) => Promise<{
+    host: string;
+    transfer: string;
+    store: HistoryChunkStore;
+} | null>;
+/** N's way into the store the envelope names. */
+export type HistoryStoreOpener = (host: string, transfer: string) => HistoryChunkStore;

package/dist/pairing/historyStore.js

@@ -0,0 +1,9 @@
+/** "network" is retried, "full" ends the snapshot early, "missing" is a chunk the store doesn't have. */
+export class HistoryStoreError extends Error {
+    code;
+    constructor(code, message = `History store: ${code}`) {
+        super(message);
+        this.name = "HistoryStoreError";
+        this.code = code;
+    }
+}

package/dist/pairing/newDevice.d.ts

@@ -1,4 +1,5 @@
 import { type PairingEmoji, type PairingServer } from "@gryt/crypto";
+import type { HistoryStoreOpener } from "./historyStore.js";
 import type { HistoryProgress, HistorySink, PairedServerDevice, PairingClock, PairingDeviceInfo, PairingEndReason, PairingOidc, PairingStorage } from "./interfaces.js";
 import type { PairingRelay } from "./relayClient.js";
 export type NewDeviceState = {
@@ -46,14 +47,21 @@ export interface NewDeviceOptions {
     /** Only for a server with its own auth server: goes in the QR for A to check against its own. */
     relayOrigin?: string;
     approvalMs?: number;
-    /** N's archive. Without it, history A sends is left on the relay. */
+    /** N's archive. Without it, history A sends is left where A put it. */
     history?: HistorySink;
+    /** Opens the server the envelope names for the history. Without it, no history comes across. */
+    historyStore?: HistoryStoreOpener;
+    /** This device's archive scope for a server. A's records are refiled under it, since the
+        desktop files by server key and the phone by host. */
+    localScope?: (host: string) => string;
 }
 export interface NewDevicePairing {
     readonly state: NewDeviceState;
     subscribe(listener: (state: NewDeviceState) => void): () => void;
     /** Null until the envelope brings a history key, or with no sink. */
     readonly history: HistoryProgress | null;
+    /** True when A had servers but sent no history, so the old messages stay on A. */
+    readonly historyLeftBehind: boolean;
     subscribeHistory(listener: (progress: HistoryProgress) => void): () => void;
     start(): void;
     /** In `joining`: N's device on each server, once its KeyPackages are up. */

package/dist/pairing/newDevice.js

@@ -3,6 +3,18 @@ import { sha256 } from "@noble/hashes/sha2.js";
 import { randomBytes } from "@noble/hashes/utils.js";
 import { mailbox, openJson, PairingEnded, reasonFor, sealJson, systemClock } from "./channel.js";
 import { createHistoryReceiver } from "./historyReceiver.js";
+function refiled(sink, servers, localScope) {
+    if (!localScope)
+        return sink;
+    const hosts = new Map(servers.map((s) => [s.scope, s.host]));
+    // Asked per batch: the desktop's scope needs the server's key, pinned only once it has joined.
+    return {
+        put: (records) => sink.put(records.map((r) => {
+            const host = hosts.get(r.scope);
+            return host === undefined ? r : { ...r, scope: localScope(host) };
+        })),
+    };
+}
 export const PAIRING_APPROVAL_MS = 60_000;
 const PAIRING_OIDC_SCOPE = "openid profile email offline_access";
 /** N: shows the QR and code, checks the emoji, opens the envelope, signs in, then says ready. */
@@ -16,6 +28,7 @@ export function createNewDevicePairing(options) {
     let current = null;
     let receiver = null;
     let history = null;
+    let historyLeftBehind = false;
     const historyListeners = new Set();
     const setHistory = (progress) => {
         history = progress;
@@ -156,12 +169,17 @@ export function createNewDevicePairing(options) {
             const envelope = outcome;
             const tokens = envelope.account ? await signIn(envelope.account, envelope.from) : null;
             await storage.commit(envelope, tokens);
-            if (envelope.history && options.history) {
-                const { id, token } = paired();
+            const held = envelope.history;
+            // History only ever comes from a server the envelope names; an older A's relay copy is left there.
+            if (held?.host && held.transfer && options.history && options.historyStore) {
                 receiver = createHistoryReceiver({
-                    relay, id, token, key: envelope.history.key, sink: options.history, clock, signal: abort.signal, onProgress: setHistory,
+                    store: options.historyStore(held.host, held.transfer),
+                    key: held.key, sink: refiled(options.history, envelope.servers, options.localScope), clock, signal: abort.signal, onProgress: setHistory,
                 });
-                receiver.listManifest(envelope.history.manifest.chunks);
+                receiver.listManifest(held.manifest.chunks);
+            }
+            else if (envelope.servers.length > 0) {
+                historyLeftBehind = true;
             }
             set({ phase: "joining", servers: envelope.servers, from: envelope.from });
         }
@@ -220,6 +238,9 @@ export function createNewDevicePairing(options) {
         get history() {
             return history;
         },
+        get historyLeftBehind() {
+            return historyLeftBehind;
+        },
         subscribeHistory(listener) {
             historyListeners.add(listener);
             return () => void historyListeners.delete(listener);

package/dist/pairing/serverHistory.d.ts

@@ -0,0 +1,26 @@
+import type { HistoryChunkStore, HistoryStoreChooser, HistoryStoreError as StoreError, HistoryStoreOpener } from "./historyStore.js";
+import type { PairingClock } from "./interfaces.js";
+/** `fetch`, cut down to what the store uses. A body here is bytes, which PairingFetch can't send. */
+export type HistoryFetch = (url: string, init: {
+    method: string;
+    headers?: Record<string, string>;
+    body?: Uint8Array;
+    signal?: AbortSignal;
+}) => Promise<{
+    status: number;
+    json(): Promise<{
+        linkHistory?: boolean;
+    } | null>;
+    arrayBuffer(): Promise<ArrayBuffer>;
+}>;
+/** How this device reaches a server: its address and, once it has joined, its token. */
+export interface ServerAccess {
+    base(host: string): string;
+    token(host: string): Promise<string | null> | string | null;
+}
+export declare function historyStoreError(status: number): StoreError;
+export declare function serverHistoryStore(access: ServerAccess, host: string, transfer: string, fetch: HistoryFetch, clock?: PairingClock): HistoryChunkStore;
+/** The first server this device is signed in to that says it holds linked devices' history. */
+export declare function chooseHistoryServer(access: ServerAccess, fetch: HistoryFetch, clock?: PairingClock): HistoryStoreChooser;
+/** The new device's side: the server the envelope names. */
+export declare function openServerHistory(access: ServerAccess, fetch: HistoryFetch, clock?: PairingClock): HistoryStoreOpener;

package/dist/pairing/serverHistory.js

@@ -0,0 +1,65 @@
+import { base64Url } from "@gryt/crypto";
+import { randomBytes } from "@noble/hashes/utils.js";
+import { HistoryStoreError } from "./historyStore.js";
+import { systemClock } from "./channel.js";
+/** The new device only gets a token once it has joined, which can trail the envelope. */
+const TOKEN_WAIT_MS = 90_000;
+export function historyStoreError(status) {
+    if (status === 404)
+        return new HistoryStoreError("missing");
+    if (status === 413 || status === 429)
+        return new HistoryStoreError("full");
+    // A stale token gets refreshed by the connection, and a server error may pass: both are worth another try.
+    if (status === 401 || (status >= 500 && status !== 503))
+        return new HistoryStoreError("network");
+    return new HistoryStoreError("refused");
+}
+export function serverHistoryStore(access, host, transfer, fetch, clock = systemClock) {
+    async function tokenFor(signal) {
+        for (const start = clock.now();; await clock.sleep(1000, signal)) {
+            const token = await access.token(host);
+            if (token)
+                return token;
+            if (clock.now() - start > TOKEN_WAIT_MS)
+                throw new HistoryStoreError("refused", `No session on ${host} to fetch history from.`);
+        }
+    }
+    async function call(method, n, signal, body) {
+        const headers = { authorization: `Bearer ${await tokenFor(signal)}` };
+        if (body)
+            headers["content-type"] = "application/octet-stream";
+        const url = `${access.base(host)}/api/link-history/${transfer}/${n}`;
+        const res = await fetch(url, { method, headers, body, signal }).catch(() => {
+            throw new HistoryStoreError("network");
+        });
+        if (res.status < 200 || res.status > 299)
+            throw historyStoreError(res.status);
+        return res;
+    }
+    return {
+        put: async (n, sealed, signal) => void (await call("PUT", n, signal, sealed)),
+        get: async (n, signal) => new Uint8Array(await (await call("GET", n, signal)).arrayBuffer()),
+        delete: async (n) => void (await call("DELETE", n)),
+    };
+}
+/** The first server this device is signed in to that says it holds linked devices' history. */
+export function chooseHistoryServer(access, fetch, clock = systemClock) {
+    return async (servers) => {
+        for (const { host } of servers) {
+            if (!(await access.token(host)))
+                continue;
+            const info = await fetch(`${access.base(host)}/info`, { method: "GET" })
+                .then((r) => (r.status === 200 ? r.json() : null))
+                .catch(() => null);
+            if (info?.linkHistory !== true)
+                continue;
+            const transfer = base64Url(randomBytes(18));
+            return { host, transfer, store: serverHistoryStore(access, host, transfer, fetch, clock) };
+        }
+        return null;
+    };
+}
+/** The new device's side: the server the envelope names. */
+export function openServerHistory(access, fetch, clock = systemClock) {
+    return (host, transfer) => serverHistoryStore(access, host, transfer, fetch, clock);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.32.1",
+  "version": "0.33.1",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {
@@ -43,7 +43,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@gryt/crypto": "0.13.0",
+    "@gryt/crypto": "0.14.0",
     "@noble/hashes": "^2.3.0"
   }
 }