@gryt/core 0.22.0 → 0.24.0

package/dist/import/discordImportJob.d.ts

@@ -0,0 +1,56 @@
+import type { SealedAttachmentKey } from "@gryt/crypto";
+import type { MlsDmContent } from "../mls/content.js";
+import type { DiscordPackageMessage } from "./discordPackage.js";
+export interface ImportFile {
+    name: string;
+    type: string;
+    bytes: Uint8Array;
+}
+/** Where downloads wait until they're uploaded: Cache Storage on the desktop, files on the phone. */
+export interface ImportFileStore {
+    has(url: string): Promise<boolean>;
+    put(url: string, file: Omit<ImportFile, "name">): Promise<void>;
+    take(url: string): Promise<Omit<ImportFile, "name"> | null>;
+    drop(url: string): Promise<void>;
+}
+/** What the job needs from the app. Each app supplies its own, and a test supplies fakes. */
+export interface ImportDeps {
+    send: (content: MlsDmContent) => Promise<void>;
+    /** Null when Discord won't hand it over (expired, gone, refused). `retryAfterMs` when it answers 429. */
+    download: (url: string) => Promise<Omit<ImportFile, "name"> | {
+        retryAfterMs: number;
+    } | null>;
+    upload: (file: ImportFile) => Promise<{
+        id: string;
+        key: SealedAttachmentKey;
+    }>;
+    store: ImportFileStore;
+    sleep: (ms: number) => Promise<void>;
+    newId: () => string;
+}
+export interface ImportProgress {
+    messages: {
+        done: number;
+        total: number;
+    };
+    downloaded: {
+        done: number;
+        total: number;
+    };
+    sent: {
+        done: number;
+        total: number;
+    };
+    missing: number;
+    bytes: number;
+    startedAt: number;
+}
+export interface ImportControl {
+    paused: () => boolean;
+    stopped: () => boolean;
+}
+/** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
+export declare function retryAfterMs(header: string | null): number;
+export declare const importId: (discordId: string) => string;
+/** Messages without files first, then every file downloaded to the store, then the messages that carry files. */
+export declare function runDiscordImport(messages: DiscordPackageMessage[], deps: ImportDeps, control: ImportControl, report: (p: ImportProgress) => void): Promise<ImportProgress>;

package/dist/import/discordImportJob.js

@@ -0,0 +1,124 @@
+import { importBatches } from "../mls/content.js";
+const DOWNLOADS_AT_ONCE = 4;
+/** `mls:send` allows about one send every two seconds sustained (RL_SEND in the server). */
+const SEND_GAP_MS = 2_000;
+const FILE_BATCH = 20;
+const DOWNLOAD_RETRIES = 5;
+const MAX_RATE_LIMIT_WAIT_MS = 60_000;
+const DEFAULT_RATE_LIMIT_WAIT_MS = 5_000;
+/** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
+export function retryAfterMs(header) {
+    const seconds = Number(header);
+    return header && seconds >= 0 ? seconds * 1000 : DEFAULT_RATE_LIMIT_WAIT_MS;
+}
+export const importId = (discordId) => `discord-${discordId}`;
+function nameOf(url) {
+    try {
+        return decodeURIComponent(new URL(url).pathname.split("/").pop() || "file");
+    }
+    catch {
+        return "file";
+    }
+}
+/** Messages without files first, then every file downloaded to the store, then the messages that carry files. */
+export async function runDiscordImport(messages, deps, control, report) {
+    const withFiles = messages.filter((m) => m.attachments.length > 0);
+    const urls = [...new Set(withFiles.flatMap((m) => m.attachments))];
+    const p = {
+        messages: { done: 0, total: messages.length },
+        downloaded: { done: 0, total: urls.length },
+        sent: { done: 0, total: urls.length },
+        missing: 0,
+        bytes: 0,
+        startedAt: Date.now(),
+    };
+    const tick = () => report({ ...p, messages: { ...p.messages }, downloaded: { ...p.downloaded }, sent: { ...p.sent } });
+    tick();
+    const hold = async () => {
+        while (control.paused() && !control.stopped())
+            await deps.sleep(500);
+    };
+    const sendAll = async (entries) => {
+        for (const batch of importBatches(entries, deps.newId)) {
+            await hold();
+            if (control.stopped())
+                return;
+            await deps.send(batch);
+            p.messages.done += batch.entries.length;
+            tick();
+            await deps.sleep(SEND_GAP_MS);
+        }
+    };
+    await sendAll(messages.filter((m) => m.attachments.length === 0).map((m) => ({ id: importId(m.discordId), sentAt: m.sentAt, text: m.text })));
+    const missing = new Set();
+    let next = 0;
+    let clearAt = 0;
+    const downloadFile = async (url) => {
+        for (let attempt = 0; attempt <= DOWNLOAD_RETRIES && !control.stopped(); attempt++) {
+            const wait = clearAt - Date.now();
+            if (wait > 0)
+                await deps.sleep(wait);
+            const got = await deps.download(url).catch(() => null);
+            if (!got || !("retryAfterMs" in got))
+                return got;
+            // Every worker waits, or the other three spend the wait getting 429s too.
+            clearAt = Math.max(clearAt, Date.now() + Math.min(got.retryAfterMs, MAX_RATE_LIMIT_WAIT_MS));
+        }
+        return null;
+    };
+    const worker = async () => {
+        while (!control.stopped()) {
+            await hold();
+            if (next >= urls.length)
+                break;
+            const url = urls[next++];
+            if (!(await deps.store.has(url))) {
+                const file = await downloadFile(url);
+                if (file) {
+                    p.bytes += file.bytes.length;
+                    await deps.store.put(url, file);
+                }
+                else {
+                    missing.add(url);
+                    p.missing = missing.size;
+                }
+            }
+            p.downloaded.done++;
+            tick();
+        }
+    };
+    await Promise.all(Array.from({ length: DOWNLOADS_AT_ONCE }, worker));
+    const uploaded = new Map();
+    let pending = [];
+    for (const m of withFiles) {
+        if (control.stopped())
+            break;
+        const attachments = {};
+        for (const url of m.attachments) {
+            await hold();
+            let up = uploaded.get(url);
+            const stored = up || missing.has(url) ? null : await deps.store.take(url);
+            if (stored) {
+                up = await deps.upload({ name: nameOf(url), ...stored });
+                uploaded.set(url, up);
+                await deps.store.drop(url);
+            }
+            if (up)
+                attachments[up.id] = up.key;
+            p.sent.done++;
+            tick();
+        }
+        const lost = m.attachments.length - Object.keys(attachments).length;
+        const note = lost ? `${lost === 1 ? "A file" : `${lost} files`} from Discord couldn’t be downloaded.` : "";
+        const text = [m.text, note].filter(Boolean).join("\n");
+        pending.push({ id: importId(m.discordId), sentAt: m.sentAt, text, ...(Object.keys(attachments).length ? { attachments } : {}) });
+        if (pending.length >= FILE_BATCH) {
+            await sendAll(pending);
+            pending = [];
+        }
+    }
+    if (pending.length)
+        await sendAll(pending);
+    tick();
+    return p;
+}

package/dist/index.d.ts

@@ -12,10 +12,12 @@ export { describeFolderRules, folderFollowNote, folderPhrase, } from "./permissi
 export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_URL, TERMS_VERSION, } from "./terms/termsAgreement.js";
 export { createMlsDmDriver, MlsDriverError, type MlsDriverErrorCode } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
+export { reencryptedId, reencryptMine, type ReencryptCandidate, type ReencryptDeps } from "./mls/reencrypt.js";
 export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, type MlsDmContent, type MlsHistoryEntry, type MlsImportEntry, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction, type MlsReaction } from "./mls/reactions.js";
 export { type DiscordPackageDm, type DiscordPackageMessage, isDiscordPackage, linksExpireAt, listDiscordDms, type PackageFiles, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";
+export { type ImportControl, type ImportDeps, type ImportFile, type ImportFileStore, importId, type ImportProgress, retryAfterMs, runDiscordImport, } from "./import/discordImportJob.js";
 export type { DmSealingMode, MlsAddOwnDeviceOptions, MlsClaimedKeyPackage, MlsDecryptedMessage, MlsDeviceRecord, MlsDeviceRef, MlsDmDriver, MlsDmDriverOptions, MlsDmEvents, MlsGroupPosition, MlsGroupRecord, MlsGroupView, MlsKeyPackageRecord, MlsListedDevice, MlsLogEntry, MlsDmDiagnostics, MlsOwnDevice, MlsOwnDeviceAdd, MlsPins, MlsRefusal, MlsReply, MlsServerCapability, MlsStateStore, MlsTransport, MlsWelcomeDelivery, } from "./mls/interfaces.js";
 export { createApproverPairing, type ApproverOptions, type ApproverPairing, type ApproverState, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, type NewDeviceOptions, type NewDevicePairing, type NewDeviceState, } from "./pairing/newDevice.js";

package/dist/index.js

@@ -14,10 +14,12 @@ export { describeFolderRules, folderFollowNote, folderPhrase, } from "./permissi
 export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_URL, TERMS_VERSION, } from "./terms/termsAgreement.js";
 export { createMlsDmDriver, MlsDriverError } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
+export { reencryptedId, reencryptMine } from "./mls/reencrypt.js";
 export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction } from "./mls/reactions.js";
 export { isDiscordPackage, linksExpireAt, listDiscordDms, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";
+export { importId, retryAfterMs, runDiscordImport, } from "./import/discordImportJob.js";
 export { createApproverPairing, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, } from "./pairing/newDevice.js";
 export { createPairingRelay, PairingRelayError } from "./pairing/relayClient.js";

package/dist/mls/reencrypt.d.ts

@@ -0,0 +1,40 @@
+import type { MlsDmContent, MlsHistoryEntry } from "./content.js";
+export declare const reencryptedId: (messageId: string) => string;
+/** The fields of a server message that decide whether it can move. */
+export interface ReencryptCandidate {
+    message_id: string;
+    sender_server_id: string;
+    text?: string | null;
+    created_at: string | Date;
+    reply_to_message_id?: string | null;
+    mls?: boolean;
+    sealed?: string | null;
+    mls_placeholder?: object | null;
+    attachments?: readonly string[] | null;
+    reactions?: readonly object[] | null;
+    poll?: object | null;
+    pinned_at?: string | Date | null;
+    thread_id?: string | null;
+}
+export interface ReencryptDeps {
+    fetchOlder: (before: string | undefined) => Promise<{
+        items: ReencryptCandidate[];
+        hasMore: boolean;
+    }>;
+    send: (content: Extract<MlsDmContent, {
+        type: "history";
+    }>) => Promise<void>;
+    /** This device's own copy, kept before the plaintext goes. */
+    keep: (entries: MlsHistoryEntry[], receivedAt: number) => Promise<void>;
+    remove: (messageId: string) => void;
+    sleep: (ms: number) => Promise<void>;
+    now: () => number;
+    newId: () => string;
+    stopped: () => boolean;
+    report: (done: number, total: number) => void;
+}
+/**
+ * Sends this person's plaintext as history over MLS, and removes each one only after its
+ * batch went. New ids, so re-running repeats nothing and `chat:deleted` can't take the copy.
+ */
+export declare function reencryptMine(deps: ReencryptDeps, me: string): Promise<number>;

package/dist/mls/reencrypt.js

@@ -0,0 +1,57 @@
+import { historyBatches } from "./content.js";
+/* `chat:delete` scores 1 and decays one per 3 s (RL_DELETE); `chat:fetch` allows 15 in 10 s. */
+const DELETE_GAP_MS = 3_100;
+const FETCH_GAP_MS = 700;
+export const reencryptedId = (messageId) => `re-${messageId}`;
+/** Plain text only: files, reactions, polls and pins can't travel in a history entry, so they stay. */
+function canMove(m, me) {
+    return m.sender_server_id === me && !!m.text && !m.mls && !m.sealed && !m.mls_placeholder
+        && !m.attachments?.length && !m.reactions?.length && !m.poll && !m.pinned_at && !m.thread_id;
+}
+async function plaintextOfMine(deps, me) {
+    const out = [];
+    let before;
+    for (;;) {
+        const { items, hasMore } = await deps.fetchOlder(before);
+        out.push(...items.filter((m) => canMove(m, me)));
+        if (!hasMore || !items.length || deps.stopped())
+            return out;
+        before = items.map((m) => new Date(m.created_at).toISOString()).sort()[0];
+        await deps.sleep(FETCH_GAP_MS);
+    }
+}
+/**
+ * Sends this person's plaintext as history over MLS, and removes each one only after its
+ * batch went. New ids, so re-running repeats nothing and `chat:deleted` can't take the copy.
+ */
+export async function reencryptMine(deps, me) {
+    const mine = await plaintextOfMine(deps, me);
+    const ids = new Set(mine.map((m) => m.message_id));
+    const originals = new Map(mine.map((m) => [reencryptedId(m.message_id), m.message_id]));
+    const entries = mine.map((m) => {
+        const reply = m.reply_to_message_id;
+        return {
+            id: reencryptedId(m.message_id),
+            senderId: me,
+            sentAt: new Date(m.created_at).getTime(),
+            text: m.text ?? "",
+            ...(reply ? { replyTo: ids.has(reply) ? reencryptedId(reply) : reply } : {}),
+        };
+    });
+    const batches = historyBatches(entries, deps.newId);
+    const total = batches.reduce((n, b) => n + b.entries.length, 0);
+    let done = 0;
+    deps.report(done, total);
+    for (const batch of batches) {
+        if (deps.stopped())
+            return done;
+        await deps.send(batch);
+        await deps.keep(batch.entries, deps.now());
+        for (const e of batch.entries) {
+            deps.remove(originals.get(e.id));
+            deps.report(++done, total);
+            await deps.sleep(DELETE_GAP_MS);
+        }
+    }
+    return done;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {