@gryt/core 0.23.0 → 0.24.0

package/dist/import/discordImportJob.d.ts

@@ -16,8 +16,10 @@ export interface ImportFileStore {
 /** What the job needs from the app. Each app supplies its own, and a test supplies fakes. */
 export interface ImportDeps {
     send: (content: MlsDmContent) => Promise<void>;
-    /** Null when Discord won't hand it over (expired, gone, refused). */
-    download: (url: string) => Promise<Omit<ImportFile, "name"> | null>;
+    /** Null when Discord won't hand it over (expired, gone, refused). `retryAfterMs` when it answers 429. */
+    download: (url: string) => Promise<Omit<ImportFile, "name"> | {
+        retryAfterMs: number;
+    } | null>;
     upload: (file: ImportFile) => Promise<{
         id: string;
         key: SealedAttachmentKey;
@@ -47,6 +49,8 @@ export interface ImportControl {
     paused: () => boolean;
     stopped: () => boolean;
 }
+/** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
+export declare function retryAfterMs(header: string | null): number;
 export declare const importId: (discordId: string) => string;
 /** Messages without files first, then every file downloaded to the store, then the messages that carry files. */
 export declare function runDiscordImport(messages: DiscordPackageMessage[], deps: ImportDeps, control: ImportControl, report: (p: ImportProgress) => void): Promise<ImportProgress>;

package/dist/import/discordImportJob.js

@@ -3,6 +3,14 @@ const DOWNLOADS_AT_ONCE = 4;
 /** `mls:send` allows about one send every two seconds sustained (RL_SEND in the server). */
 const SEND_GAP_MS = 2_000;
 const FILE_BATCH = 20;
+const DOWNLOAD_RETRIES = 5;
+const MAX_RATE_LIMIT_WAIT_MS = 60_000;
+const DEFAULT_RATE_LIMIT_WAIT_MS = 5_000;
+/** For a 429's Retry-After header. Only the seconds form: anything else gets the default wait. */
+export function retryAfterMs(header) {
+    const seconds = Number(header);
+    return header && seconds >= 0 ? seconds * 1000 : DEFAULT_RATE_LIMIT_WAIT_MS;
+}
 export const importId = (discordId) => `discord-${discordId}`;
 function nameOf(url) {
     try {
@@ -44,6 +52,20 @@ export async function runDiscordImport(messages, deps, control, report) {
     await sendAll(messages.filter((m) => m.attachments.length === 0).map((m) => ({ id: importId(m.discordId), sentAt: m.sentAt, text: m.text })));
     const missing = new Set();
     let next = 0;
+    let clearAt = 0;
+    const downloadFile = async (url) => {
+        for (let attempt = 0; attempt <= DOWNLOAD_RETRIES && !control.stopped(); attempt++) {
+            const wait = clearAt - Date.now();
+            if (wait > 0)
+                await deps.sleep(wait);
+            const got = await deps.download(url).catch(() => null);
+            if (!got || !("retryAfterMs" in got))
+                return got;
+            // Every worker waits, or the other three spend the wait getting 429s too.
+            clearAt = Math.max(clearAt, Date.now() + Math.min(got.retryAfterMs, MAX_RATE_LIMIT_WAIT_MS));
+        }
+        return null;
+    };
     const worker = async () => {
         while (!control.stopped()) {
             await hold();
@@ -51,7 +73,7 @@ export async function runDiscordImport(messages, deps, control, report) {
                 break;
             const url = urls[next++];
             if (!(await deps.store.has(url))) {
-                const file = await deps.download(url).catch(() => null);
+                const file = await downloadFile(url);
                 if (file) {
                     p.bytes += file.bytes.length;
                     await deps.store.put(url, file);

package/dist/index.d.ts

@@ -12,11 +12,12 @@ export { describeFolderRules, folderFollowNote, folderPhrase, } from "./permissi
 export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_URL, TERMS_VERSION, } from "./terms/termsAgreement.js";
 export { createMlsDmDriver, MlsDriverError, type MlsDriverErrorCode } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
+export { reencryptedId, reencryptMine, type ReencryptCandidate, type ReencryptDeps } from "./mls/reencrypt.js";
 export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, type MlsDmContent, type MlsHistoryEntry, type MlsImportEntry, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction, type MlsReaction } from "./mls/reactions.js";
 export { type DiscordPackageDm, type DiscordPackageMessage, isDiscordPackage, linksExpireAt, listDiscordDms, type PackageFiles, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";
-export { type ImportControl, type ImportDeps, type ImportFile, type ImportFileStore, importId, type ImportProgress, runDiscordImport, } from "./import/discordImportJob.js";
+export { type ImportControl, type ImportDeps, type ImportFile, type ImportFileStore, importId, type ImportProgress, retryAfterMs, runDiscordImport, } from "./import/discordImportJob.js";
 export type { DmSealingMode, MlsAddOwnDeviceOptions, MlsClaimedKeyPackage, MlsDecryptedMessage, MlsDeviceRecord, MlsDeviceRef, MlsDmDriver, MlsDmDriverOptions, MlsDmEvents, MlsGroupPosition, MlsGroupRecord, MlsGroupView, MlsKeyPackageRecord, MlsListedDevice, MlsLogEntry, MlsDmDiagnostics, MlsOwnDevice, MlsOwnDeviceAdd, MlsPins, MlsRefusal, MlsReply, MlsServerCapability, MlsStateStore, MlsTransport, MlsWelcomeDelivery, } from "./mls/interfaces.js";
 export { createApproverPairing, type ApproverOptions, type ApproverPairing, type ApproverState, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, type NewDeviceOptions, type NewDevicePairing, type NewDeviceState, } from "./pairing/newDevice.js";

package/dist/index.js

@@ -14,11 +14,12 @@ export { describeFolderRules, folderFollowNote, folderPhrase, } from "./permissi
 export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_URL, TERMS_VERSION, } from "./terms/termsAgreement.js";
 export { createMlsDmDriver, MlsDriverError } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
+export { reencryptedId, reencryptMine } from "./mls/reencrypt.js";
 export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction } from "./mls/reactions.js";
 export { isDiscordPackage, linksExpireAt, listDiscordDms, readDiscordDm, } from "./import/discordPackage.js";
 export { inflateRaw, openZipBytes } from "./import/unzip.js";
-export { importId, runDiscordImport, } from "./import/discordImportJob.js";
+export { importId, retryAfterMs, runDiscordImport, } from "./import/discordImportJob.js";
 export { createApproverPairing, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, } from "./pairing/newDevice.js";
 export { createPairingRelay, PairingRelayError } from "./pairing/relayClient.js";

package/dist/mls/reencrypt.d.ts

@@ -0,0 +1,40 @@
+import type { MlsDmContent, MlsHistoryEntry } from "./content.js";
+export declare const reencryptedId: (messageId: string) => string;
+/** The fields of a server message that decide whether it can move. */
+export interface ReencryptCandidate {
+    message_id: string;
+    sender_server_id: string;
+    text?: string | null;
+    created_at: string | Date;
+    reply_to_message_id?: string | null;
+    mls?: boolean;
+    sealed?: string | null;
+    mls_placeholder?: object | null;
+    attachments?: readonly string[] | null;
+    reactions?: readonly object[] | null;
+    poll?: object | null;
+    pinned_at?: string | Date | null;
+    thread_id?: string | null;
+}
+export interface ReencryptDeps {
+    fetchOlder: (before: string | undefined) => Promise<{
+        items: ReencryptCandidate[];
+        hasMore: boolean;
+    }>;
+    send: (content: Extract<MlsDmContent, {
+        type: "history";
+    }>) => Promise<void>;
+    /** This device's own copy, kept before the plaintext goes. */
+    keep: (entries: MlsHistoryEntry[], receivedAt: number) => Promise<void>;
+    remove: (messageId: string) => void;
+    sleep: (ms: number) => Promise<void>;
+    now: () => number;
+    newId: () => string;
+    stopped: () => boolean;
+    report: (done: number, total: number) => void;
+}
+/**
+ * Sends this person's plaintext as history over MLS, and removes each one only after its
+ * batch went. New ids, so re-running repeats nothing and `chat:deleted` can't take the copy.
+ */
+export declare function reencryptMine(deps: ReencryptDeps, me: string): Promise<number>;

package/dist/mls/reencrypt.js

@@ -0,0 +1,57 @@
+import { historyBatches } from "./content.js";
+/* `chat:delete` scores 1 and decays one per 3 s (RL_DELETE); `chat:fetch` allows 15 in 10 s. */
+const DELETE_GAP_MS = 3_100;
+const FETCH_GAP_MS = 700;
+export const reencryptedId = (messageId) => `re-${messageId}`;
+/** Plain text only: files, reactions, polls and pins can't travel in a history entry, so they stay. */
+function canMove(m, me) {
+    return m.sender_server_id === me && !!m.text && !m.mls && !m.sealed && !m.mls_placeholder
+        && !m.attachments?.length && !m.reactions?.length && !m.poll && !m.pinned_at && !m.thread_id;
+}
+async function plaintextOfMine(deps, me) {
+    const out = [];
+    let before;
+    for (;;) {
+        const { items, hasMore } = await deps.fetchOlder(before);
+        out.push(...items.filter((m) => canMove(m, me)));
+        if (!hasMore || !items.length || deps.stopped())
+            return out;
+        before = items.map((m) => new Date(m.created_at).toISOString()).sort()[0];
+        await deps.sleep(FETCH_GAP_MS);
+    }
+}
+/**
+ * Sends this person's plaintext as history over MLS, and removes each one only after its
+ * batch went. New ids, so re-running repeats nothing and `chat:deleted` can't take the copy.
+ */
+export async function reencryptMine(deps, me) {
+    const mine = await plaintextOfMine(deps, me);
+    const ids = new Set(mine.map((m) => m.message_id));
+    const originals = new Map(mine.map((m) => [reencryptedId(m.message_id), m.message_id]));
+    const entries = mine.map((m) => {
+        const reply = m.reply_to_message_id;
+        return {
+            id: reencryptedId(m.message_id),
+            senderId: me,
+            sentAt: new Date(m.created_at).getTime(),
+            text: m.text ?? "",
+            ...(reply ? { replyTo: ids.has(reply) ? reencryptedId(reply) : reply } : {}),
+        };
+    });
+    const batches = historyBatches(entries, deps.newId);
+    const total = batches.reduce((n, b) => n + b.entries.length, 0);
+    let done = 0;
+    deps.report(done, total);
+    for (const batch of batches) {
+        if (deps.stopped())
+            return done;
+        await deps.send(batch);
+        await deps.keep(batch.entries, deps.now());
+        for (const e of batch.entries) {
+            deps.remove(originals.get(e.id));
+            deps.report(++done, total);
+            await deps.sleep(DELETE_GAP_MS);
+        }
+    }
+    return done;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.23.0",
+  "version": "0.24.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {