@gryt/core 0.27.0 → 0.29.0

package/dist/import/discordImportJob.d.ts

@@ -20,15 +20,22 @@ export interface ImportDeps {
     download: (url: string) => Promise<Omit<ImportFile, "name"> | {
         retryAfterMs: number;
     } | null>;
+    /** `retryAfterMs` when the server answers 429, `tooLarge` when it refuses the file's size. */
     upload: (file: ImportFile) => Promise<{
         id: string;
         key: SealedAttachmentKey;
+    } | {
+        retryAfterMs: number;
+    } | {
+        tooLarge: true;
     }>;
     store: ImportFileStore;
     sleep: (ms: number) => Promise<void>;
     newId: () => string;
     /** Downloads and uploads running together. A phone holding each file in memory wants fewer. */
     transfersAtOnce?: number;
+    /** Send a big import's text as one sealed upload. Only for apps that read `import_bundle`. */
+    bundles?: boolean;
 }
 export interface ImportProgress {
     messages: {
@@ -44,6 +51,8 @@ export interface ImportProgress {
         total: number;
     };
     missing: number;
+    /** Files this server won't take, each left out with a note on its message. */
+    tooLarge: number;
     bytes: number;
     startedAt: number;
 }

package/dist/import/discordImportJob.js

@@ -1,9 +1,13 @@
-import { importBatches } from "../mls/content.js";
+import { encodeImportBundle, importBatches } from "../mls/content.js";
 const TRANSFERS_AT_ONCE = 16;
 /** `mls:send` allows about one send every two seconds sustained (RL_SEND in the server). */
 const SEND_GAP_MS = 2_000;
 const FILE_BATCH = 20;
+/* A batch holds up to 200, and one send is the point where a bundle starts saving time. */
+const BUNDLE_ABOVE = 200;
 const DOWNLOAD_RETRIES = 5;
+/* A server allows 30 uploads a minute with a ban after, so a big import waits many times over. */
+const UPLOAD_RETRIES = 200;
 const MAX_RATE_LIMIT_WAIT_MS = 60_000;
 const DEFAULT_RATE_LIMIT_WAIT_MS = 5_000;
 /** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
@@ -29,6 +33,7 @@ export async function runDiscordImport(messages, deps, control, report) {
         downloaded: { done: 0, total: urls.length },
         sent: { done: 0, total: urls.length },
         missing: 0,
+        tooLarge: 0,
         bytes: 0,
         startedAt: Date.now(),
     };
@@ -38,6 +43,32 @@ export async function runDiscordImport(messages, deps, control, report) {
         while (control.paused() && !control.stopped())
             await deps.sleep(500);
     };
+    let uploadClearAt = 0;
+    const uploadFile = async (file) => {
+        for (let attempt = 0; attempt <= UPLOAD_RETRIES && !control.stopped(); attempt++) {
+            const wait = uploadClearAt - Date.now();
+            if (wait > 0)
+                await deps.sleep(wait);
+            const got = await deps.upload(file);
+            if (!("retryAfterMs" in got))
+                return got;
+            uploadClearAt = Math.max(uploadClearAt, Date.now() + Math.min(got.retryAfterMs, MAX_RATE_LIMIT_WAIT_MS));
+        }
+        if (control.stopped())
+            return null;
+        throw new Error("The server kept refusing uploads.");
+    };
+    /* False when the bundle can't go up, so the batches carry it. */
+    const sendBundle = async (entries) => {
+        await hold();
+        const up = await uploadFile({ name: "discord-import.json", type: "application/json", bytes: encodeImportBundle(entries) });
+        if (!up || "tooLarge" in up)
+            return false;
+        await deps.send({ type: "import_bundle", id: deps.newId(), source: "discord", attachments: { [up.id]: up.key } });
+        p.messages.done += entries.length;
+        tick();
+        return true;
+    };
     const sendAll = async (entries) => {
         for (const batch of importBatches(entries, deps.newId)) {
             await hold();
@@ -49,7 +80,9 @@ export async function runDiscordImport(messages, deps, control, report) {
             await deps.sleep(SEND_GAP_MS);
         }
     };
-    await sendAll(messages.filter((m) => m.attachments.length === 0).map((m) => ({ id: importId(m.discordId), sentAt: m.sentAt, text: m.text })));
+    const textOnly = messages.filter((m) => m.attachments.length === 0).map((m) => ({ id: importId(m.discordId), sentAt: m.sentAt, text: m.text }));
+    if (!(deps.bundles && textOnly.length > BUNDLE_ABOVE && !control.stopped() && (await sendBundle(textOnly))))
+        await sendAll(textOnly);
     const missing = new Set();
     let clearAt = 0;
     const downloadFile = async (url) => {
@@ -92,11 +125,20 @@ export async function runDiscordImport(messages, deps, control, report) {
         tick();
     });
     const uploaded = new Map();
+    const tooLarge = new Set();
     await eachAtOnce(async (url) => {
         const stored = missing.has(url) ? null : await deps.store.take(url);
         if (stored) {
-            uploaded.set(url, await deps.upload({ name: nameOf(url), ...stored }));
-            await deps.store.drop(url);
+            const up = await uploadFile({ name: nameOf(url), ...stored });
+            if (up && "tooLarge" in up) {
+                tooLarge.add(url);
+                p.tooLarge = tooLarge.size;
+            }
+            else if (up) {
+                uploaded.set(url, up);
+            }
+            if (up)
+                await deps.store.drop(url);
         }
         p.sent.done++;
         tick();
@@ -111,9 +153,11 @@ export async function runDiscordImport(messages, deps, control, report) {
             if (up)
                 attachments[up.id] = up.key;
         }
-        const lost = m.attachments.length - Object.keys(attachments).length;
-        const note = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
-        const text = [m.text, note].filter(Boolean).join("\n");
+        const big = m.attachments.filter((u) => tooLarge.has(u)).length;
+        const lost = m.attachments.length - Object.keys(attachments).length - big;
+        const lostNote = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
+        const bigNote = big ? `${big === 1 ? "A file" : `${big} files`} from Discord ${big === 1 ? "was" : "were"} too big for this server.` : "";
+        const text = [m.text, lostNote, bigNote].filter(Boolean).join("\n");
         pending.push({ id: importId(m.discordId), sentAt: m.sentAt, text, ...(Object.keys(attachments).length ? { attachments } : {}) });
         if (pending.length >= FILE_BATCH) {
             await sendAll(pending);

package/dist/index.d.ts

@@ -13,7 +13,7 @@ export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_
 export { createMlsDmDriver, MlsDriverError, type MlsDriverErrorCode } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
 export { reencryptedId, reencryptMine, type ReencryptCandidate, type ReencryptDeps } from "./mls/reencrypt.js";
-export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, type MlsDmContent, type MlsHistoryEntry, type MlsHistoryReaction, type MlsImportEntry, readMlsDmContent, } from "./mls/content.js";
+export { decodeMlsDmContent, encodeImportBundle, readImportBundle, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, type MlsDmContent, type MlsHistoryEntry, type MlsHistoryReaction, type MlsImportEntry, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction, type MlsReaction } from "./mls/reactions.js";
 export { type DiscordPackageDm, type DiscordPackageMessage, isDiscordPackage, linksExpireAt, listDiscordDms, type PackageFiles, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";

package/dist/index.js

@@ -15,7 +15,7 @@ export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_
 export { createMlsDmDriver, MlsDriverError } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
 export { reencryptedId, reencryptMine } from "./mls/reencrypt.js";
-export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, readMlsDmContent, } from "./mls/content.js";
+export { decodeMlsDmContent, encodeImportBundle, readImportBundle, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction } from "./mls/reactions.js";
 export { isDiscordPackage, linksExpireAt, listDiscordDms, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";

package/dist/mls/content.d.ts

@@ -30,6 +30,13 @@ export type MlsDmContent = {
     id: string;
     source: "discord";
     entries: MlsImportEntry[];
+} | {
+    /** An import too big for messages: its entries sit in one sealed upload, read with `readImportBundle`. */
+    type: "import_bundle";
+    id: string;
+    source: "discord";
+    /** Upload id to its key. Exactly one. */
+    attachments: Record<string, SealedAttachmentKey>;
 } | {
     type: "history";
     /** For the batch itself, so a repeat is the same message. */
@@ -82,6 +89,9 @@ export declare function acceptImport(content: Extract<MlsDmContent, {
     senderId: string;
     at: number;
 }): MlsHistoryEntry[];
+export declare function encodeImportBundle(entries: MlsImportEntry[]): Uint8Array;
+/** A bundle's entries, or null for junk. A peer wrote it, so each entry is checked like a message's. */
+export declare function readImportBundle(bytes: Uint8Array): MlsImportEntry[] | null;
 /** Null for anything this version can't read: a newer version, a new type, or junk. */
 export declare function decodeMlsDmContent(bytes: Uint8Array): MlsDmContent | null;
 /**

package/dist/mls/content.js

@@ -84,7 +84,10 @@ function readAttachments(v) {
 /** The same bytes the phone writes: `v` first, then the content's fields in this order. */
 export function encodeMlsDmContent(content) {
     let body;
-    if (content.type === "history") {
+    if (content.type === "import_bundle") {
+        body = { type: content.type, id: content.id, source: content.source, attachments: content.attachments };
+    }
+    else if (content.type === "history") {
         body = { type: content.type, id: content.id, entries: content.entries.map(historyBody) };
     }
     else if (content.type === "import") {
@@ -213,6 +216,17 @@ function readImportEntry(v) {
     }
     return out;
 }
+export function encodeImportBundle(entries) {
+    return new TextEncoder().encode(JSON.stringify({ v: MLS_DM_CONTENT_VERSION, entries: entries.map(importBody) }));
+}
+/** A bundle's entries, or null for junk. A peer wrote it, so each entry is checked like a message's. */
+export function readImportBundle(bytes) {
+    const raw = parse(bytes);
+    if (!isObject(raw) || raw.v !== MLS_DM_CONTENT_VERSION || !Array.isArray(raw.entries))
+        return null;
+    const entries = raw.entries.map(readImportEntry);
+    return entries.every((e) => e !== null) ? entries : null;
+}
 /** Null for anything this version can't read: a newer version, a new type, or junk. */
 export function decodeMlsDmContent(bytes) {
     return fromRaw(parse(bytes));
@@ -225,7 +239,7 @@ export function readMlsDmContent(bytes) {
     const raw = parse(bytes);
     return fromRaw(raw) ?? (isNewer(raw) ? "newer" : null);
 }
-const KNOWN_TYPES = new Set(["message", "edit", "delete", "reaction", "history", "import"]);
+const KNOWN_TYPES = new Set(["message", "edit", "delete", "reaction", "history", "import", "import_bundle"]);
 const TYPE_NAME = /^[a-z][a-z0-9_]{0,31}$/;
 /** A later version, or a type this one doesn't know that names its message the way ours do. */
 function isNewer(raw) {
@@ -259,6 +273,10 @@ function fromRaw(raw) {
         const entries = raw.entries.map(readImportEntry);
         return entries.every((e) => e !== null) ? { type, id, source: raw.source, entries } : null;
     }
+    if (type === "import_bundle") {
+        const attachments = raw.source === "discord" ? readAttachments(raw.attachments) : null;
+        return attachments && Object.keys(attachments).length === 1 ? { type, id, source: "discord", attachments } : null;
+    }
     if (type === "message" && isText(text)) {
         const out = { type, id, text };
         if (raw.replyTo !== undefined) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.27.0",
+  "version": "0.29.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {