@gryt/core 0.27.0 → 0.28.0

package/dist/import/discordImportJob.d.ts

@@ -20,9 +20,14 @@ export interface ImportDeps {
     download: (url: string) => Promise<Omit<ImportFile, "name"> | {
         retryAfterMs: number;
     } | null>;
+    /** `retryAfterMs` when the server answers 429, `tooLarge` when it refuses the file's size. */
     upload: (file: ImportFile) => Promise<{
         id: string;
         key: SealedAttachmentKey;
+    } | {
+        retryAfterMs: number;
+    } | {
+        tooLarge: true;
     }>;
     store: ImportFileStore;
     sleep: (ms: number) => Promise<void>;
@@ -44,6 +49,8 @@ export interface ImportProgress {
         total: number;
     };
     missing: number;
+    /** Files this server won't take, each left out with a note on its message. */
+    tooLarge: number;
     bytes: number;
     startedAt: number;
 }

package/dist/import/discordImportJob.js

@@ -4,6 +4,8 @@ const TRANSFERS_AT_ONCE = 16;
 const SEND_GAP_MS = 2_000;
 const FILE_BATCH = 20;
 const DOWNLOAD_RETRIES = 5;
+/* A server allows 30 uploads a minute with a ban after, so a big import waits many times over. */
+const UPLOAD_RETRIES = 200;
 const MAX_RATE_LIMIT_WAIT_MS = 60_000;
 const DEFAULT_RATE_LIMIT_WAIT_MS = 5_000;
 /** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
@@ -29,6 +31,7 @@ export async function runDiscordImport(messages, deps, control, report) {
         downloaded: { done: 0, total: urls.length },
         sent: { done: 0, total: urls.length },
         missing: 0,
+        tooLarge: 0,
         bytes: 0,
         startedAt: Date.now(),
     };
@@ -92,11 +95,35 @@ export async function runDiscordImport(messages, deps, control, report) {
         tick();
     });
     const uploaded = new Map();
+    const tooLarge = new Set();
+    let uploadClearAt = 0;
+    const uploadFile = async (file) => {
+        for (let attempt = 0; attempt <= UPLOAD_RETRIES && !control.stopped(); attempt++) {
+            const wait = uploadClearAt - Date.now();
+            if (wait > 0)
+                await deps.sleep(wait);
+            const got = await deps.upload(file);
+            if (!("retryAfterMs" in got))
+                return got;
+            uploadClearAt = Math.max(uploadClearAt, Date.now() + Math.min(got.retryAfterMs, MAX_RATE_LIMIT_WAIT_MS));
+        }
+        if (control.stopped())
+            return null;
+        throw new Error("The server kept refusing uploads.");
+    };
     await eachAtOnce(async (url) => {
         const stored = missing.has(url) ? null : await deps.store.take(url);
         if (stored) {
-            uploaded.set(url, await deps.upload({ name: nameOf(url), ...stored }));
-            await deps.store.drop(url);
+            const up = await uploadFile({ name: nameOf(url), ...stored });
+            if (up && "tooLarge" in up) {
+                tooLarge.add(url);
+                p.tooLarge = tooLarge.size;
+            }
+            else if (up) {
+                uploaded.set(url, up);
+            }
+            if (up)
+                await deps.store.drop(url);
         }
         p.sent.done++;
         tick();
@@ -111,9 +138,11 @@ export async function runDiscordImport(messages, deps, control, report) {
             if (up)
                 attachments[up.id] = up.key;
         }
-        const lost = m.attachments.length - Object.keys(attachments).length;
-        const note = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
-        const text = [m.text, note].filter(Boolean).join("\n");
+        const big = m.attachments.filter((u) => tooLarge.has(u)).length;
+        const lost = m.attachments.length - Object.keys(attachments).length - big;
+        const lostNote = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
+        const bigNote = big ? `${big === 1 ? "A file" : `${big} files`} from Discord ${big === 1 ? "was" : "were"} too big for this server.` : "";
+        const text = [m.text, lostNote, bigNote].filter(Boolean).join("\n");
         pending.push({ id: importId(m.discordId), sentAt: m.sentAt, text, ...(Object.keys(attachments).length ? { attachments } : {}) });
         if (pending.length >= FILE_BATCH) {
             await sendAll(pending);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {