@gryt/core 0.25.0 → 0.27.0

package/dist/import/discordImportJob.d.ts

@@ -27,6 +27,8 @@ export interface ImportDeps {
     store: ImportFileStore;
     sleep: (ms: number) => Promise<void>;
     newId: () => string;
+    /** Downloads and uploads running together. A phone holding each file in memory wants fewer. */
+    transfersAtOnce?: number;
 }
 export interface ImportProgress {
     messages: {

package/dist/import/discordImportJob.js

@@ -1,5 +1,5 @@
 import { importBatches } from "../mls/content.js";
-const DOWNLOADS_AT_ONCE = 4;
+const TRANSFERS_AT_ONCE = 16;
 /** `mls:send` allows about one send every two seconds sustained (RL_SEND in the server). */
 const SEND_GAP_MS = 2_000;
 const FILE_BATCH = 20;
@@ -51,7 +51,6 @@ export async function runDiscordImport(messages, deps, control, report) {
     };
     await sendAll(messages.filter((m) => m.attachments.length === 0).map((m) => ({ id: importId(m.discordId), sentAt: m.sentAt, text: m.text })));
     const missing = new Set();
-    let next = 0;
     let clearAt = 0;
     const downloadFile = async (url) => {
         for (let attempt = 0; attempt <= DOWNLOAD_RETRIES && !control.stopped(); attempt++) {
@@ -66,47 +65,51 @@ export async function runDiscordImport(messages, deps, control, report) {
         }
         return null;
     };
-    const worker = async () => {
-        while (!control.stopped()) {
-            await hold();
-            if (next >= urls.length)
-                break;
-            const url = urls[next++];
-            if (!(await deps.store.has(url))) {
-                const file = await downloadFile(url);
-                if (file) {
-                    p.bytes += file.bytes.length;
-                    await deps.store.put(url, file);
-                }
-                else {
-                    missing.add(url);
-                    p.missing = missing.size;
-                }
+    const eachAtOnce = async (run) => {
+        let next = 0;
+        await Promise.all(Array.from({ length: deps.transfersAtOnce ?? TRANSFERS_AT_ONCE }, async () => {
+            while (!control.stopped()) {
+                await hold();
+                if (next >= urls.length)
+                    break;
+                await run(urls[next++]);
             }
-            p.downloaded.done++;
-            tick();
-        }
+        }));
     };
-    await Promise.all(Array.from({ length: DOWNLOADS_AT_ONCE }, worker));
+    await eachAtOnce(async (url) => {
+        if (!(await deps.store.has(url))) {
+            const file = await downloadFile(url);
+            if (file) {
+                p.bytes += file.bytes.length;
+                await deps.store.put(url, file);
+            }
+            else {
+                missing.add(url);
+                p.missing = missing.size;
+            }
+        }
+        p.downloaded.done++;
+        tick();
+    });
     const uploaded = new Map();
+    await eachAtOnce(async (url) => {
+        const stored = missing.has(url) ? null : await deps.store.take(url);
+        if (stored) {
+            uploaded.set(url, await deps.upload({ name: nameOf(url), ...stored }));
+            await deps.store.drop(url);
+        }
+        p.sent.done++;
+        tick();
+    });
     let pending = [];
     for (const m of withFiles) {
         if (control.stopped())
             break;
         const attachments = {};
         for (const url of m.attachments) {
-            await hold();
-            let up = uploaded.get(url);
-            const stored = up || missing.has(url) ? null : await deps.store.take(url);
-            if (stored) {
-                up = await deps.upload({ name: nameOf(url), ...stored });
-                uploaded.set(url, up);
-                await deps.store.drop(url);
-            }
+            const up = uploaded.get(url);
             if (up)
                 attachments[up.id] = up.key;
-            p.sent.done++;
-            tick();
         }
         const lost = m.attachments.length - Object.keys(attachments).length;
         const note = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";

package/dist/import/discordPackage.d.ts

@@ -32,5 +32,7 @@ export declare function parseDiscordTime(v: unknown): number | null;
 export declare function listDiscordDms(files: PackageFiles): Promise<DiscordPackageDm[]>;
 /** One DM's messages, oldest first. */
 export declare function readDiscordDm(files: PackageFiles, channelId: string): Promise<DiscordPackageMessage[]>;
+/** Discord's own markup as text a person can read. A custom emoji can't come over, so it keeps its name. */
+export declare function readableDiscordText(text: string, people: ReadonlyMap<string, string>, channels: Readonly<Record<string, unknown>>): string;
 /** The earliest `ex` (hex Unix seconds) among signed CDN links, or null when none carry one. */
 export declare function linksExpireAt(urls: string[]): number | null;

package/dist/import/discordPackage.js

@@ -189,7 +189,41 @@ export async function listDiscordDms(files) {
 export async function readDiscordDm(files, channelId) {
     const idx = indexNames(files.names);
     const dir = channelFolder(idx, channelId);
-    return dir ? messagesIn(files, idx, dir) : [];
+    if (!dir)
+        return [];
+    const names = await peopleIn(files, idx, channelId);
+    const channels = ((await readJson(files, idx, "messages/index.json")) ?? {});
+    return (await messagesIn(files, idx, dir)).map((m) => ({ ...m, text: readableDiscordText(m.text, names, channels) }));
+}
+async function peopleIn(files, idx, channelId) {
+    const names = new Map();
+    const user = (await readJson(files, idx, "account/user.json"));
+    const add = (u) => {
+        const name = u?.global_name || u?.username;
+        if (typeof u?.id === "string" && typeof name === "string")
+            names.set(u.id, name);
+    };
+    add(user);
+    for (const rel of Array.isArray(user?.relationships) ? user.relationships : [])
+        add(rel?.user);
+    const channel = (await readJson(files, idx, `messages/c${channelId}/channel.json`));
+    const label = (await readJson(files, idx, "messages/index.json"))?.[channelId];
+    const peer = Array.isArray(channel?.recipients) ? channel.recipients.find((id) => id !== user?.id) : undefined;
+    if (typeof peer === "string" && typeof label === "string" && !names.has(peer))
+        names.set(peer, displayName(label));
+    return names;
+}
+/** Discord's own markup as text a person can read. A custom emoji can't come over, so it keeps its name. */
+export function readableDiscordText(text, people, channels) {
+    return text
+        .replace(/<a?:(\w+):\d+>/g, ":$1:")
+        .replace(/<@!?(\d+)>/g, (_, id) => `@${people.get(id) ?? "someone"}`)
+        .replace(/<@&\d+>/g, "@role")
+        .replace(/<#(\d+)>/g, (_, id) => {
+        const label = channels[id];
+        const name = typeof label === "string" && !/^unknown/i.test(label) ? label.split(",")[0].trim() : "channel";
+        return `#${name}`;
+    });
 }
 /** The earliest `ex` (hex Unix seconds) among signed CDN links, or null when none carry one. */
 export function linksExpireAt(urls) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.25.0",
+  "version": "0.27.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {