@gryt/core 0.31.0 → 0.32.1

package/dist/import/discordImportJob.d.ts

@@ -41,6 +41,14 @@ export interface ImportDeps {
     /** The server's file size limit, so a file over it is never sealed and sent to be refused. */
     maxFileBytes?: number | null;
 }
+/** A file that didn't come over, for the receipt. */
+export interface ImportLeftOut {
+    url: string;
+    name: string;
+    /** Bytes, when it was downloaded. */
+    size: number | null;
+    reason: "too_large" | "not_downloaded";
+}
 export interface ImportProgress {
     messages: {
         done: number;
@@ -58,6 +66,7 @@ export interface ImportProgress {
     /** Files this server won't take, each left out with a note on its message. */
     tooLarge: number;
     bytes: number;
+    leftOut: ImportLeftOut[];
     startedAt: number;
 }
 export interface ImportControl {
@@ -67,5 +76,10 @@ export interface ImportControl {
 /** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
 export declare function retryAfterMs(header: string | null): number;
 export declare const importId: (discordId: string) => string;
+/**
+ * Another go at files left out: new entries beside the originals, holding only those files.
+ * A first run's message is already in history, and a repeat of its id is skipped there.
+ */
+export declare function retryLeftOut(messages: DiscordPackageMessage[], urls: ReadonlySet<string>): DiscordPackageMessage[];
 /** Text first, then 20 messages at a time: their files go straight through, then those messages go. */
 export declare function runDiscordImport(messages: DiscordPackageMessage[], deps: ImportDeps, control: ImportControl, report: (p: ImportProgress) => void): Promise<ImportProgress>;

package/dist/import/discordImportJob.js

@@ -24,6 +24,16 @@ function nameOf(url) {
         return "file";
     }
 }
+/**
+ * Another go at files left out: new entries beside the originals, holding only those files.
+ * A first run's message is already in history, and a repeat of its id is skipped there.
+ */
+export function retryLeftOut(messages, urls) {
+    return messages.flatMap((m) => {
+        const attachments = m.attachments.filter((u) => urls.has(u));
+        return attachments.length ? [{ ...m, discordId: `${m.discordId}-files`, text: "", attachments }] : [];
+    });
+}
 /** Text first, then 20 messages at a time: their files go straight through, then those messages go. */
 export async function runDiscordImport(messages, deps, control, report) {
     const fresh = [];
@@ -39,9 +49,10 @@ export async function runDiscordImport(messages, deps, control, report) {
         missing: 0,
         tooLarge: 0,
         bytes: 0,
+        leftOut: [],
         startedAt: Date.now(),
     };
-    const tick = () => report({ ...p, messages: { ...p.messages }, downloaded: { ...p.downloaded }, sent: { ...p.sent } });
+    const tick = () => report({ ...p, leftOut: [...p.leftOut], messages: { ...p.messages }, downloaded: { ...p.downloaded }, sent: { ...p.sent } });
     tick();
     const hold = async () => {
         while (control.paused() && !control.stopped())
@@ -118,6 +129,7 @@ export async function runDiscordImport(messages, deps, control, report) {
             else if (!control.stopped()) {
                 missing.add(url);
                 p.missing = missing.size;
+                p.leftOut.push({ url, name: nameOf(url), size: null, reason: "not_downloaded" });
             }
         }
         if (control.stopped()) {
@@ -143,6 +155,7 @@ export async function runDiscordImport(messages, deps, control, report) {
             if ("tooLarge" in up) {
                 tooLarge.add(url);
                 p.tooLarge = tooLarge.size;
+                p.leftOut.push({ url, name: nameOf(url), size: file.bytes.length, reason: "too_large" });
             }
             else
                 uploaded.set(url, up);

package/dist/index.d.ts

@@ -17,7 +17,7 @@ export { decodeMlsDmContent, encodeImportBundle, readImportBundle, encodeMlsDmCo
 export { applyMlsReaction, mlsReactionAction, type MlsReaction } from "./mls/reactions.js";
 export { type DiscordPackageDm, type DiscordPackageMessage, isDiscordPackage, linksExpireAt, listDiscordDms, type PackageFiles, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";
-export { type ImportControl, type ImportDeps, type ImportFile, type ImportFileStore, importId, type ImportProgress, retryAfterMs, runDiscordImport, } from "./import/discordImportJob.js";
+export { type ImportControl, type ImportDeps, type ImportFile, type ImportFileStore, importId, type ImportProgress, retryAfterMs, retryLeftOut, runDiscordImport, type ImportLeftOut, } from "./import/discordImportJob.js";
 export type { DmSealingMode, MlsAddOwnDeviceOptions, MlsClaimedKeyPackage, MlsDecryptedMessage, MlsDeviceRecord, MlsDeviceRef, MlsDmDriver, MlsDmDriverOptions, MlsDmEvents, MlsGroupPosition, MlsGroupRecord, MlsGroupView, MlsKeyPackageRecord, MlsListedDevice, MlsLogEntry, MlsDmDiagnostics, MlsOwnDevice, MlsOwnDeviceAdd, MlsPins, MlsRefusal, MlsReply, MlsServerCapability, MlsStateStore, MlsTransport, MlsWelcomeDelivery, } from "./mls/interfaces.js";
 export { createApproverPairing, type ApproverOptions, type ApproverPairing, type ApproverState, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, type NewDeviceOptions, type NewDevicePairing, type NewDeviceState, } from "./pairing/newDevice.js";

package/dist/index.js

@@ -19,7 +19,7 @@ export { decodeMlsDmContent, encodeImportBundle, readImportBundle, encodeMlsDmCo
 export { applyMlsReaction, mlsReactionAction } from "./mls/reactions.js";
 export { isDiscordPackage, linksExpireAt, listDiscordDms, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";
-export { importId, retryAfterMs, runDiscordImport, } from "./import/discordImportJob.js";
+export { importId, retryAfterMs, retryLeftOut, runDiscordImport, } from "./import/discordImportJob.js";
 export { createApproverPairing, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, } from "./pairing/newDevice.js";
 export { createPairingRelay, PairingRelayError } from "./pairing/relayClient.js";

package/dist/mls/dmDriver.js

@@ -2,6 +2,7 @@ import { addMlsMembers, asIdentityScope, base64UrlDecode, createMlsGroup, decode
 /** Devices per person per server. Must match the server's MLS_MAX_DEVICES, or extra devices are left out quietly. */
 const MLS_MAX_DEVICES_PER_PERSON = 100;
 const COMMIT_ATTEMPTS = 5;
+const OWN_DEVICE_ADDS_AT_ONCE = 4;
 /** socket.io closes the connection on a packet with more than 10 binary parts (GRYT-1522). */
 const MAX_BINARY_PARTS = 10;
 const LOG_PAGE = MAX_BINARY_PARTS;
@@ -1072,21 +1073,29 @@ export function createMlsDmDriver(options) {
         const ids = (await store.listGroups()).map((r) => r.conversationId);
         const first = (options.order ?? []).filter((id) => ids.includes(id));
         const ordered = [...new Set([...first, ...ids])];
-        const results = [];
-        for (const conversationId of ordered) {
-            let result;
-            try {
-                result = await run(conversationId, () => addOwnDeviceTo(conversationId, deviceId, ownPersonKey));
-            }
-            catch (e) {
-                if (e instanceof MlsDriverError && e.code === "not_own_device")
-                    throw e;
-                const groupId = (await store.loadGroup(conversationId))?.groupId ?? "";
-                result = { conversationId, groupId, outcome: "failed", add: null, error: e instanceof MlsDriverError ? e.code : "error" };
+        const results = new Array(ordered.length);
+        let done = 0;
+        let next = 0;
+        // Each group is its own commit, so a few go at once; `order` still decides which start first.
+        const worker = async () => {
+            while (next < ordered.length) {
+                const i = next++;
+                const conversationId = ordered[i];
+                let result;
+                try {
+                    result = await run(conversationId, () => addOwnDeviceTo(conversationId, deviceId, ownPersonKey));
+                }
+                catch (e) {
+                    if (e instanceof MlsDriverError && e.code === "not_own_device")
+                        throw e;
+                    const groupId = (await store.loadGroup(conversationId))?.groupId ?? "";
+                    result = { conversationId, groupId, outcome: "failed", add: null, error: e instanceof MlsDriverError ? e.code : "error" };
+                }
+                results[i] = result;
+                options.onProgress?.({ done: ++done, total: ordered.length, result });
             }
-            results.push(result);
-            options.onProgress?.({ done: results.length, total: ordered.length, result });
-        }
+        };
+        await Promise.all(Array.from({ length: Math.min(OWN_DEVICE_ADDS_AT_ONCE, ordered.length) }, worker));
         return results;
     }
     return {

package/dist/pairing/approver.js

@@ -142,16 +142,30 @@ export function createApproverPairing(options) {
     }
     async function addEverywhere(ready) {
         const added = {};
-        for (const { host, deviceId } of ready) {
+        // Servers go at once, so progress is the sum over them.
+        const counts = new Map();
+        const report = (host) => {
+            let done = 0;
+            let total = 0;
+            for (const c of counts.values()) {
+                done += c.done;
+                total += c.total;
+            }
+            set({ phase: "adding", device: confirmed().device, host, done, total });
+        };
+        await Promise.all(ready.map(async ({ host, deviceId }) => {
             const driver = devices(host);
             if (!driver)
-                continue;
+                return;
             for (let tries = 0;; tries++) {
                 try {
                     added[host] = await driver.addOwnDevice(deviceId, {
-                        onProgress: ({ done, total }) => set({ phase: "adding", device: confirmed().device, host, done, total }),
+                        onProgress: ({ done, total }) => {
+                            counts.set(host, { done, total });
+                            report(host);
+                        },
                     });
-                    break;
+                    return;
                 }
                 catch (e) {
                     // The server lists N's device once its KeyPackages are in, which can trail N's "ready".
@@ -160,7 +174,7 @@ export function createApproverPairing(options) {
                     await clock.sleep(2000, abort.signal);
                 }
             }
-        }
+        }));
         return added;
     }
     /** Each group's cursor on every server in the envelope: the snapshot covers up to there. */

package/dist/pairing/newDevice.js

@@ -234,7 +234,14 @@ export function createNewDevicePairing(options) {
                 throw new Error("Not waiting to join.");
             const { from } = state;
             const body = sealJson(current.session, { t: "ready", devices: devices.map(({ host, deviceId }) => ({ host, deviceId })) });
-            await relay.post(current.id, current.token, { type: "sealed", body });
+            try {
+                await relay.post(current.id, current.token, { type: "sealed", body });
+            }
+            catch (e) {
+                // A closed before N got here: say so, rather than sit on "joining" with nothing listening.
+                await end(reasonFor(e));
+                throw e;
+            }
             set({ phase: "linked", from });
             void linked(from);
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.31.0",
+  "version": "0.32.1",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {