@gryt/core 0.26.0 → 0.28.0

package/dist/import/discordImportJob.d.ts

@@ -20,9 +20,14 @@ export interface ImportDeps {
     download: (url: string) => Promise<Omit<ImportFile, "name"> | {
         retryAfterMs: number;
     } | null>;
+    /** `retryAfterMs` when the server answers 429, `tooLarge` when it refuses the file's size. */
     upload: (file: ImportFile) => Promise<{
         id: string;
         key: SealedAttachmentKey;
+    } | {
+        retryAfterMs: number;
+    } | {
+        tooLarge: true;
     }>;
     store: ImportFileStore;
     sleep: (ms: number) => Promise<void>;
@@ -44,6 +49,8 @@ export interface ImportProgress {
         total: number;
     };
     missing: number;
+    /** Files this server won't take, each left out with a note on its message. */
+    tooLarge: number;
     bytes: number;
     startedAt: number;
 }

package/dist/import/discordImportJob.js

@@ -4,6 +4,8 @@ const TRANSFERS_AT_ONCE = 16;
 const SEND_GAP_MS = 2_000;
 const FILE_BATCH = 20;
 const DOWNLOAD_RETRIES = 5;
+/* A server allows 30 uploads a minute with a ban after, so a big import waits many times over. */
+const UPLOAD_RETRIES = 200;
 const MAX_RATE_LIMIT_WAIT_MS = 60_000;
 const DEFAULT_RATE_LIMIT_WAIT_MS = 5_000;
 /** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
@@ -29,6 +31,7 @@ export async function runDiscordImport(messages, deps, control, report) {
         downloaded: { done: 0, total: urls.length },
         sent: { done: 0, total: urls.length },
         missing: 0,
+        tooLarge: 0,
         bytes: 0,
         startedAt: Date.now(),
     };
@@ -92,11 +95,35 @@ export async function runDiscordImport(messages, deps, control, report) {
         tick();
     });
     const uploaded = new Map();
+    const tooLarge = new Set();
+    let uploadClearAt = 0;
+    const uploadFile = async (file) => {
+        for (let attempt = 0; attempt <= UPLOAD_RETRIES && !control.stopped(); attempt++) {
+            const wait = uploadClearAt - Date.now();
+            if (wait > 0)
+                await deps.sleep(wait);
+            const got = await deps.upload(file);
+            if (!("retryAfterMs" in got))
+                return got;
+            uploadClearAt = Math.max(uploadClearAt, Date.now() + Math.min(got.retryAfterMs, MAX_RATE_LIMIT_WAIT_MS));
+        }
+        if (control.stopped())
+            return null;
+        throw new Error("The server kept refusing uploads.");
+    };
     await eachAtOnce(async (url) => {
         const stored = missing.has(url) ? null : await deps.store.take(url);
         if (stored) {
-            uploaded.set(url, await deps.upload({ name: nameOf(url), ...stored }));
-            await deps.store.drop(url);
+            const up = await uploadFile({ name: nameOf(url), ...stored });
+            if (up && "tooLarge" in up) {
+                tooLarge.add(url);
+                p.tooLarge = tooLarge.size;
+            }
+            else if (up) {
+                uploaded.set(url, up);
+            }
+            if (up)
+                await deps.store.drop(url);
         }
         p.sent.done++;
         tick();
@@ -111,9 +138,11 @@ export async function runDiscordImport(messages, deps, control, report) {
             if (up)
                 attachments[up.id] = up.key;
         }
-        const lost = m.attachments.length - Object.keys(attachments).length;
-        const note = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
-        const text = [m.text, note].filter(Boolean).join("\n");
+        const big = m.attachments.filter((u) => tooLarge.has(u)).length;
+        const lost = m.attachments.length - Object.keys(attachments).length - big;
+        const lostNote = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
+        const bigNote = big ? `${big === 1 ? "A file" : `${big} files`} from Discord ${big === 1 ? "was" : "were"} too big for this server.` : "";
+        const text = [m.text, lostNote, bigNote].filter(Boolean).join("\n");
         pending.push({ id: importId(m.discordId), sentAt: m.sentAt, text, ...(Object.keys(attachments).length ? { attachments } : {}) });
         if (pending.length >= FILE_BATCH) {
             await sendAll(pending);

package/dist/import/discordPackage.d.ts

@@ -32,5 +32,7 @@ export declare function parseDiscordTime(v: unknown): number | null;
 export declare function listDiscordDms(files: PackageFiles): Promise<DiscordPackageDm[]>;
 /** One DM's messages, oldest first. */
 export declare function readDiscordDm(files: PackageFiles, channelId: string): Promise<DiscordPackageMessage[]>;
+/** Discord's own markup as text a person can read. A custom emoji can't come over, so it keeps its name. */
+export declare function readableDiscordText(text: string, people: ReadonlyMap<string, string>, channels: Readonly<Record<string, unknown>>): string;
 /** The earliest `ex` (hex Unix seconds) among signed CDN links, or null when none carry one. */
 export declare function linksExpireAt(urls: string[]): number | null;

package/dist/import/discordPackage.js

@@ -189,7 +189,41 @@ export async function listDiscordDms(files) {
 export async function readDiscordDm(files, channelId) {
     const idx = indexNames(files.names);
     const dir = channelFolder(idx, channelId);
-    return dir ? messagesIn(files, idx, dir) : [];
+    if (!dir)
+        return [];
+    const names = await peopleIn(files, idx, channelId);
+    const channels = ((await readJson(files, idx, "messages/index.json")) ?? {});
+    return (await messagesIn(files, idx, dir)).map((m) => ({ ...m, text: readableDiscordText(m.text, names, channels) }));
+}
+async function peopleIn(files, idx, channelId) {
+    const names = new Map();
+    const user = (await readJson(files, idx, "account/user.json"));
+    const add = (u) => {
+        const name = u?.global_name || u?.username;
+        if (typeof u?.id === "string" && typeof name === "string")
+            names.set(u.id, name);
+    };
+    add(user);
+    for (const rel of Array.isArray(user?.relationships) ? user.relationships : [])
+        add(rel?.user);
+    const channel = (await readJson(files, idx, `messages/c${channelId}/channel.json`));
+    const label = (await readJson(files, idx, "messages/index.json"))?.[channelId];
+    const peer = Array.isArray(channel?.recipients) ? channel.recipients.find((id) => id !== user?.id) : undefined;
+    if (typeof peer === "string" && typeof label === "string" && !names.has(peer))
+        names.set(peer, displayName(label));
+    return names;
+}
+/** Discord's own markup as text a person can read. A custom emoji can't come over, so it keeps its name. */
+export function readableDiscordText(text, people, channels) {
+    return text
+        .replace(/<a?:(\w+):\d+>/g, ":$1:")
+        .replace(/<@!?(\d+)>/g, (_, id) => `@${people.get(id) ?? "someone"}`)
+        .replace(/<@&\d+>/g, "@role")
+        .replace(/<#(\d+)>/g, (_, id) => {
+        const label = channels[id];
+        const name = typeof label === "string" && !/^unknown/i.test(label) ? label.split(",")[0].trim() : "channel";
+        return `#${name}`;
+    });
 }
 /** The earliest `ex` (hex Unix seconds) among signed CDN links, or null when none carry one. */
 export function linksExpireAt(urls) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.26.0",
+  "version": "0.28.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {