@gryt/core 0.28.0 → 0.30.0

package/dist/import/discordImportJob.d.ts

@@ -34,6 +34,12 @@ export interface ImportDeps {
     newId: () => string;
     /** Downloads and uploads running together. A phone holding each file in memory wants fewer. */
     transfersAtOnce?: number;
+    /** Send a big import's text as one sealed upload. Only for apps that read `import_bundle`. */
+    bundles?: boolean;
+    /** Whether this device's history already holds an entry, so a second run skips it. */
+    alreadyHas?: (entryId: string) => Promise<boolean>;
+    /** The server's file size limit, so a file over it is never sealed and sent to be refused. */
+    maxFileBytes?: number | null;
 }
 export interface ImportProgress {
     messages: {
@@ -61,5 +67,5 @@ export interface ImportControl {
 /** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
 export declare function retryAfterMs(header: string | null): number;
 export declare const importId: (discordId: string) => string;
-/** Messages without files first, then every file downloaded to the store, then the messages that carry files. */
+/** Text first, then 20 messages at a time: their files go straight through, then those messages go. */
 export declare function runDiscordImport(messages: DiscordPackageMessage[], deps: ImportDeps, control: ImportControl, report: (p: ImportProgress) => void): Promise<ImportProgress>;

package/dist/import/discordImportJob.js

@@ -1,8 +1,10 @@
-import { importBatches } from "../mls/content.js";
+import { encodeImportBundle, importBatches } from "../mls/content.js";
 const TRANSFERS_AT_ONCE = 16;
 /** `mls:send` allows about one send every two seconds sustained (RL_SEND in the server). */
 const SEND_GAP_MS = 2_000;
 const FILE_BATCH = 20;
+/* A batch holds up to 200, and one send is the point where a bundle starts saving time. */
+const BUNDLE_ABOVE = 200;
 const DOWNLOAD_RETRIES = 5;
 /* A server allows 30 uploads a minute with a ban after, so a big import waits many times over. */
 const UPLOAD_RETRIES = 200;
@@ -22,12 +24,16 @@ function nameOf(url) {
         return "file";
     }
 }
-/** Messages without files first, then every file downloaded to the store, then the messages that carry files. */
+/** Text first, then 20 messages at a time: their files go straight through, then those messages go. */
 export async function runDiscordImport(messages, deps, control, report) {
-    const withFiles = messages.filter((m) => m.attachments.length > 0);
+    const fresh = [];
+    for (const m of messages)
+        if (!(await deps.alreadyHas?.(importId(m.discordId))))
+            fresh.push(m);
+    const withFiles = fresh.filter((m) => m.attachments.length > 0);
     const urls = [...new Set(withFiles.flatMap((m) => m.attachments))];
     const p = {
-        messages: { done: 0, total: messages.length },
+        messages: { done: messages.length - fresh.length, total: messages.length },
         downloaded: { done: 0, total: urls.length },
         sent: { done: 0, total: urls.length },
         missing: 0,
@@ -41,6 +47,32 @@ export async function runDiscordImport(messages, deps, control, report) {
         while (control.paused() && !control.stopped())
             await deps.sleep(500);
     };
+    let uploadClearAt = 0;
+    const uploadFile = async (file) => {
+        for (let attempt = 0; attempt <= UPLOAD_RETRIES && !control.stopped(); attempt++) {
+            const wait = uploadClearAt - Date.now();
+            if (wait > 0)
+                await deps.sleep(wait);
+            const got = await deps.upload(file);
+            if (!("retryAfterMs" in got))
+                return got;
+            uploadClearAt = Math.max(uploadClearAt, Date.now() + Math.min(got.retryAfterMs, MAX_RATE_LIMIT_WAIT_MS));
+        }
+        if (control.stopped())
+            return null;
+        throw new Error("The server kept refusing uploads.");
+    };
+    /* False when the bundle can't go up, so the batches carry it. */
+    const sendBundle = async (entries) => {
+        await hold();
+        const up = await uploadFile({ name: "discord-import.json", type: "application/json", bytes: encodeImportBundle(entries) });
+        if (!up || "tooLarge" in up)
+            return false;
+        await deps.send({ type: "import_bundle", id: deps.newId(), source: "discord", attachments: { [up.id]: up.key } });
+        p.messages.done += entries.length;
+        tick();
+        return true;
+    };
     const sendAll = async (entries) => {
         for (const batch of importBatches(entries, deps.newId)) {
             await hold();
@@ -52,7 +84,9 @@ export async function runDiscordImport(messages, deps, control, report) {
             await deps.sleep(SEND_GAP_MS);
         }
     };
-    await sendAll(messages.filter((m) => m.attachments.length === 0).map((m) => ({ id: importId(m.discordId), sentAt: m.sentAt, text: m.text })));
+    const textOnly = fresh.filter((m) => m.attachments.length === 0).map((m) => ({ id: importId(m.discordId), sentAt: m.sentAt, text: m.text }));
+    if (!(deps.bundles && textOnly.length > BUNDLE_ABOVE && !control.stopped() && (await sendBundle(textOnly))))
+        await sendAll(textOnly);
     const missing = new Set();
     let clearAt = 0;
     const downloadFile = async (url) => {
@@ -68,89 +102,85 @@ export async function runDiscordImport(messages, deps, control, report) {
         }
         return null;
     };
-    const eachAtOnce = async (run) => {
-        let next = 0;
-        await Promise.all(Array.from({ length: deps.transfersAtOnce ?? TRANSFERS_AT_ONCE }, async () => {
-            while (!control.stopped()) {
-                await hold();
-                if (next >= urls.length)
-                    break;
-                await run(urls[next++]);
-            }
-        }));
-    };
-    await eachAtOnce(async (url) => {
-        if (!(await deps.store.has(url))) {
-            const file = await downloadFile(url);
-            if (file) {
+    const uploaded = new Map();
+    const tooLarge = new Set();
+    const finished = new Set();
+    /* The file is only kept if it can't go up now, so a stop or a failure never downloads it twice. */
+    const transfer = async (url) => {
+        if (finished.has(url))
+            return;
+        finished.add(url);
+        let file = await deps.store.take(url);
+        if (!file) {
+            file = await downloadFile(url);
+            if (file)
                 p.bytes += file.bytes.length;
-                await deps.store.put(url, file);
-            }
-            else {
+            else if (!control.stopped()) {
                 missing.add(url);
                 p.missing = missing.size;
             }
         }
+        if (control.stopped()) {
+            if (file)
+                await deps.store.put(url, file);
+            return;
+        }
         p.downloaded.done++;
         tick();
-    });
-    const uploaded = new Map();
-    const tooLarge = new Set();
-    let uploadClearAt = 0;
-    const uploadFile = async (file) => {
-        for (let attempt = 0; attempt <= UPLOAD_RETRIES && !control.stopped(); attempt++) {
-            const wait = uploadClearAt - Date.now();
-            if (wait > 0)
-                await deps.sleep(wait);
-            const got = await deps.upload(file);
-            if (!("retryAfterMs" in got))
-                return got;
-            uploadClearAt = Math.max(uploadClearAt, Date.now() + Math.min(got.retryAfterMs, MAX_RATE_LIMIT_WAIT_MS));
-        }
-        if (control.stopped())
-            return null;
-        throw new Error("The server kept refusing uploads.");
-    };
-    await eachAtOnce(async (url) => {
-        const stored = missing.has(url) ? null : await deps.store.take(url);
-        if (stored) {
-            const up = await uploadFile({ name: nameOf(url), ...stored });
-            if (up && "tooLarge" in up) {
+        if (file) {
+            let up;
+            try {
+                up = deps.maxFileBytes && file.bytes.length > deps.maxFileBytes ? { tooLarge: true } : await uploadFile({ name: nameOf(url), ...file });
+            }
+            catch (e) {
+                await deps.store.put(url, file);
+                throw e;
+            }
+            if (!up) {
+                await deps.store.put(url, file);
+                return;
+            }
+            if ("tooLarge" in up) {
                 tooLarge.add(url);
                 p.tooLarge = tooLarge.size;
             }
-            else if (up) {
+            else
                 uploaded.set(url, up);
-            }
-            if (up)
-                await deps.store.drop(url);
+            await deps.store.drop(url);
         }
         p.sent.done++;
         tick();
-    });
-    let pending = [];
-    for (const m of withFiles) {
+    };
+    const atOnce = deps.transfersAtOnce ?? TRANSFERS_AT_ONCE;
+    for (let i = 0; i < withFiles.length && !control.stopped(); i += FILE_BATCH) {
+        const chunk = withFiles.slice(i, i + FILE_BATCH);
+        const todo = [...new Set(chunk.flatMap((m) => m.attachments))];
+        let next = 0;
+        await Promise.all(Array.from({ length: atOnce }, async () => {
+            while (!control.stopped()) {
+                await hold();
+                if (next >= todo.length)
+                    break;
+                await transfer(todo[next++]);
+            }
+        }));
         if (control.stopped())
             break;
-        const attachments = {};
-        for (const url of m.attachments) {
-            const up = uploaded.get(url);
-            if (up)
-                attachments[up.id] = up.key;
-        }
-        const big = m.attachments.filter((u) => tooLarge.has(u)).length;
-        const lost = m.attachments.length - Object.keys(attachments).length - big;
-        const lostNote = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
-        const bigNote = big ? `${big === 1 ? "A file" : `${big} files`} from Discord ${big === 1 ? "was" : "were"} too big for this server.` : "";
-        const text = [m.text, lostNote, bigNote].filter(Boolean).join("\n");
-        pending.push({ id: importId(m.discordId), sentAt: m.sentAt, text, ...(Object.keys(attachments).length ? { attachments } : {}) });
-        if (pending.length >= FILE_BATCH) {
-            await sendAll(pending);
-            pending = [];
-        }
+        await sendAll(chunk.map((m) => {
+            const attachments = {};
+            for (const url of m.attachments) {
+                const up = uploaded.get(url);
+                if (up)
+                    attachments[up.id] = up.key;
+            }
+            const big = m.attachments.filter((u) => tooLarge.has(u)).length;
+            const lost = m.attachments.length - Object.keys(attachments).length - big;
+            const lostNote = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
+            const bigNote = big ? `${big === 1 ? "A file" : `${big} files`} from Discord ${big === 1 ? "was" : "were"} too big for this server.` : "";
+            const text = [m.text, lostNote, bigNote].filter(Boolean).join("\n");
+            return { id: importId(m.discordId), sentAt: m.sentAt, text, ...(Object.keys(attachments).length ? { attachments } : {}) };
+        }));
     }
-    if (pending.length)
-        await sendAll(pending);
     tick();
     return p;
 }

package/dist/index.d.ts

@@ -13,7 +13,7 @@ export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_
 export { createMlsDmDriver, MlsDriverError, type MlsDriverErrorCode } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
 export { reencryptedId, reencryptMine, type ReencryptCandidate, type ReencryptDeps } from "./mls/reencrypt.js";
-export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, type MlsDmContent, type MlsHistoryEntry, type MlsHistoryReaction, type MlsImportEntry, readMlsDmContent, } from "./mls/content.js";
+export { decodeMlsDmContent, encodeImportBundle, readImportBundle, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, type MlsDmContent, type MlsHistoryEntry, type MlsHistoryReaction, type MlsImportEntry, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction, type MlsReaction } from "./mls/reactions.js";
 export { type DiscordPackageDm, type DiscordPackageMessage, isDiscordPackage, linksExpireAt, listDiscordDms, type PackageFiles, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";

package/dist/index.js

@@ -15,7 +15,7 @@ export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_
 export { createMlsDmDriver, MlsDriverError } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
 export { reencryptedId, reencryptMine } from "./mls/reencrypt.js";
-export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, readMlsDmContent, } from "./mls/content.js";
+export { decodeMlsDmContent, encodeImportBundle, readImportBundle, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction } from "./mls/reactions.js";
 export { isDiscordPackage, linksExpireAt, listDiscordDms, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";

package/dist/mls/content.d.ts

@@ -30,6 +30,13 @@ export type MlsDmContent = {
     id: string;
     source: "discord";
     entries: MlsImportEntry[];
+} | {
+    /** An import too big for messages: its entries sit in one sealed upload, read with `readImportBundle`. */
+    type: "import_bundle";
+    id: string;
+    source: "discord";
+    /** Upload id to its key. Exactly one. */
+    attachments: Record<string, SealedAttachmentKey>;
 } | {
     type: "history";
     /** For the batch itself, so a repeat is the same message. */
@@ -82,6 +89,9 @@ export declare function acceptImport(content: Extract<MlsDmContent, {
     senderId: string;
     at: number;
 }): MlsHistoryEntry[];
+export declare function encodeImportBundle(entries: MlsImportEntry[]): Uint8Array;
+/** A bundle's entries, or null for junk. A peer wrote it, so each entry is checked like a message's. */
+export declare function readImportBundle(bytes: Uint8Array): MlsImportEntry[] | null;
 /** Null for anything this version can't read: a newer version, a new type, or junk. */
 export declare function decodeMlsDmContent(bytes: Uint8Array): MlsDmContent | null;
 /**

package/dist/mls/content.js

@@ -84,7 +84,10 @@ function readAttachments(v) {
 /** The same bytes the phone writes: `v` first, then the content's fields in this order. */
 export function encodeMlsDmContent(content) {
     let body;
-    if (content.type === "history") {
+    if (content.type === "import_bundle") {
+        body = { type: content.type, id: content.id, source: content.source, attachments: content.attachments };
+    }
+    else if (content.type === "history") {
         body = { type: content.type, id: content.id, entries: content.entries.map(historyBody) };
     }
     else if (content.type === "import") {
@@ -213,6 +216,17 @@ function readImportEntry(v) {
     }
     return out;
 }
+export function encodeImportBundle(entries) {
+    return new TextEncoder().encode(JSON.stringify({ v: MLS_DM_CONTENT_VERSION, entries: entries.map(importBody) }));
+}
+/** A bundle's entries, or null for junk. A peer wrote it, so each entry is checked like a message's. */
+export function readImportBundle(bytes) {
+    const raw = parse(bytes);
+    if (!isObject(raw) || raw.v !== MLS_DM_CONTENT_VERSION || !Array.isArray(raw.entries))
+        return null;
+    const entries = raw.entries.map(readImportEntry);
+    return entries.every((e) => e !== null) ? entries : null;
+}
 /** Null for anything this version can't read: a newer version, a new type, or junk. */
 export function decodeMlsDmContent(bytes) {
     return fromRaw(parse(bytes));
@@ -225,7 +239,7 @@ export function readMlsDmContent(bytes) {
     const raw = parse(bytes);
     return fromRaw(raw) ?? (isNewer(raw) ? "newer" : null);
 }
-const KNOWN_TYPES = new Set(["message", "edit", "delete", "reaction", "history", "import"]);
+const KNOWN_TYPES = new Set(["message", "edit", "delete", "reaction", "history", "import", "import_bundle"]);
 const TYPE_NAME = /^[a-z][a-z0-9_]{0,31}$/;
 /** A later version, or a type this one doesn't know that names its message the way ours do. */
 function isNewer(raw) {
@@ -259,6 +273,10 @@ function fromRaw(raw) {
         const entries = raw.entries.map(readImportEntry);
         return entries.every((e) => e !== null) ? { type, id, source: raw.source, entries } : null;
     }
+    if (type === "import_bundle") {
+        const attachments = raw.source === "discord" ? readAttachments(raw.attachments) : null;
+        return attachments && Object.keys(attachments).length === 1 ? { type, id, source: "discord", attachments } : null;
+    }
     if (type === "message" && isText(text)) {
         const out = { type, id, text };
         if (raw.replyTo !== undefined) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.28.0",
+  "version": "0.30.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {