@gryt/core 0.20.0 → 0.21.0

package/dist/import/discordPackage.d.ts

@@ -0,0 +1,36 @@
+/** The files in a package, however it was opened: a zip, a folder, a test. */
+export interface PackageFiles {
+    names: string[];
+    read(name: string): Promise<string>;
+}
+export interface DiscordPackageDm {
+    channelId: string;
+    /** Their Discord name, from the package's own index. */
+    name: string;
+    userId: string | null;
+    /** Discord's CDN serves avatars by hash without a token. Only friends carry one. */
+    avatarUrl: string | null;
+    messages: number;
+    files: number;
+    firstAt: number;
+    lastAt: number;
+}
+export interface DiscordPackageMessage {
+    discordId: string;
+    sentAt: number;
+    text: string;
+    attachments: string[];
+}
+export declare function isDiscordPackage(names: string[]): boolean;
+/** Like JSON.parse, but survives what Discord writes: raw newlines in strings, trailing commas. */
+export declare function parseLooseJson(text: string): unknown;
+/** RFC 4180: quoted fields may hold commas, quotes ("") and newlines. First row is the header. */
+export declare function parseCsv(text: string): Record<string, string>[];
+/** Discord writes "2023-04-12 21:14:05" in UTC, sometimes with fractions and an offset. */
+export declare function parseDiscordTime(v: unknown): number | null;
+/** Every one-to-one DM in the package that has at least one message, most messages first. */
+export declare function listDiscordDms(files: PackageFiles): Promise<DiscordPackageDm[]>;
+/** One DM's messages, oldest first. */
+export declare function readDiscordDm(files: PackageFiles, channelId: string): Promise<DiscordPackageMessage[]>;
+/** The earliest `ex` (hex Unix seconds) among signed CDN links, or null when none carry one. */
+export declare function linksExpireAt(urls: string[]): number | null;

package/dist/import/discordPackage.js

@@ -0,0 +1,206 @@
+/* Reads the DMs out of Discord's own data package (GRYT-1965). The package only holds
+   messages its owner sent, and its JSON is not always valid, so every read is forgiving. */
+function indexNames(names) {
+    const out = new Map();
+    const lower = names.map((n) => n.replace(/\\/g, "/").toLowerCase());
+    // A zip made by hand can wrap everything in one folder; Discord's own doesn't.
+    const roots = new Set(lower.map((n) => n.split("/")[0]));
+    const strip = roots.size === 1 && !lower.some((n) => n === "readme.txt") ? `${[...roots][0]}/` : "";
+    lower.forEach((n, i) => out.set(strip && n.startsWith(strip) ? n.slice(strip.length) : n, names[i]));
+    return out;
+}
+export function isDiscordPackage(names) {
+    const idx = indexNames(names);
+    return idx.has("messages/index.json") && idx.has("account/user.json");
+}
+const ESCAPES = { "\n": "\\n", "\r": "\\r", "\t": "\\t" };
+/** Like JSON.parse, but survives what Discord writes: raw newlines in strings, trailing commas. */
+export function parseLooseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        let out = "";
+        let inString = false;
+        for (let i = 0; i < text.length; i++) {
+            const c = text[i];
+            if (inString) {
+                if (c === "\\") {
+                    out += c + (text[++i] ?? "");
+                    continue;
+                }
+                if (c === "\"")
+                    inString = false;
+                out += ESCAPES[c] ?? c;
+            }
+            else {
+                if (c === "\"")
+                    inString = true;
+                out += c;
+            }
+        }
+        return JSON.parse(out.replace(/,\s*([}\]])/g, "$1"));
+    }
+}
+/** RFC 4180: quoted fields may hold commas, quotes ("") and newlines. First row is the header. */
+export function parseCsv(text) {
+    const rows = [];
+    let row = [];
+    let field = "";
+    let quoted = false;
+    for (let i = 0; i < text.length; i++) {
+        const c = text[i];
+        if (quoted) {
+            if (c === "\"" && text[i + 1] === "\"") {
+                field += "\"";
+                i++;
+            }
+            else if (c === "\"")
+                quoted = false;
+            else
+                field += c;
+        }
+        else if (c === "\"")
+            quoted = true;
+        else if (c === ",") {
+            row.push(field);
+            field = "";
+        }
+        else if (c === "\n" || c === "\r") {
+            if (c === "\r" && text[i + 1] === "\n")
+                i++;
+            row.push(field);
+            field = "";
+            if (row.some((f) => f !== ""))
+                rows.push(row);
+            row = [];
+        }
+        else
+            field += c;
+    }
+    row.push(field);
+    if (row.some((f) => f !== ""))
+        rows.push(row);
+    const [head, ...body] = rows;
+    if (!head)
+        return [];
+    return body.map((r) => Object.fromEntries(head.map((h, i) => [h.trim(), r[i] ?? ""])));
+}
+/** Discord writes "2023-04-12 21:14:05" in UTC, sometimes with fractions and an offset. */
+export function parseDiscordTime(v) {
+    if (typeof v !== "string" || !v.trim())
+        return null;
+    let s = v.trim().replace(" ", "T");
+    if (!/(Z|[+-]\d\d:?\d\d)$/.test(s))
+        s += "Z";
+    const t = Date.parse(s);
+    return Number.isFinite(t) ? t : null;
+}
+const field = (row, name) => {
+    const key = Object.keys(row).find((k) => k.toLowerCase() === name);
+    return key === undefined ? undefined : row[key];
+};
+function attachmentsOf(v) {
+    let list = [];
+    if (Array.isArray(v))
+        list = v;
+    else if (typeof v === "string")
+        list = v.split(/\s+/);
+    return list.filter((u) => typeof u === "string" && /^https:\/\//.test(u));
+}
+function toMessage(row) {
+    if (typeof row !== "object" || row === null)
+        return null;
+    const r = row;
+    const id = field(r, "id");
+    const sentAt = parseDiscordTime(field(r, "timestamp"));
+    const text = field(r, "contents");
+    if ((typeof id !== "string" && typeof id !== "number") || sentAt === null)
+        return null;
+    return { discordId: String(id), sentAt, text: typeof text === "string" ? text : "", attachments: attachmentsOf(field(r, "attachments")) };
+}
+async function readJson(files, idx, path) {
+    const real = idx.get(path);
+    return real === undefined ? undefined : parseLooseJson(await files.read(real));
+}
+/** Newer packages name folders `c<id>`, older ones just `<id>`. */
+function channelFolder(idx, channelId) {
+    for (const dir of [`messages/c${channelId}/`, `messages/${channelId}/`]) {
+        if (idx.has(`${dir}channel.json`))
+            return dir;
+    }
+    return null;
+}
+async function messagesIn(files, idx, dir) {
+    const json = idx.get(`${dir}messages.json`);
+    const csv = idx.get(`${dir}messages.csv`);
+    let rows = [];
+    if (json !== undefined) {
+        const parsed = parseLooseJson(await files.read(json));
+        rows = Array.isArray(parsed) ? parsed : [];
+    }
+    else if (csv !== undefined) {
+        rows = parseCsv(await files.read(csv));
+    }
+    return rows.map(toMessage).filter((m) => m !== null).sort((a, b) => a.sentAt - b.sentAt);
+}
+const displayName = (raw) => raw.replace(/^Direct Message with\s*/i, "").replace(/#\d+$/, "").trim();
+/** Every one-to-one DM in the package that has at least one message, most messages first. */
+export async function listDiscordDms(files) {
+    const idx = indexNames(files.names);
+    const user = (await readJson(files, idx, "account/user.json"));
+    const selfId = typeof user?.id === "string" ? user.id : null;
+    const avatars = new Map();
+    for (const rel of Array.isArray(user?.relationships) ? user.relationships : []) {
+        const u = rel?.user;
+        if (typeof u?.id === "string" && typeof u.avatar === "string" && /^(a_)?[0-9a-f]{32}$/i.test(u.avatar))
+            avatars.set(u.id, u.avatar);
+    }
+    const index = ((await readJson(files, idx, "messages/index.json")) ?? {});
+    const out = [];
+    for (const [channelId, label] of Object.entries(index)) {
+        const dir = channelFolder(idx, channelId);
+        if (!dir)
+            continue;
+        const channel = (await readJson(files, idx, `${dir}channel.json`));
+        const recipients = Array.isArray(channel?.recipients) ? channel.recipients.filter((r) => typeof r === "string") : [];
+        const isDm = channel?.type === 1 || channel?.type === "DM" || (!channel?.guild && recipients.length === 2);
+        if (!isDm)
+            continue;
+        const messages = await messagesIn(files, idx, dir);
+        if (messages.length === 0)
+            continue;
+        const userId = recipients.find((r) => r !== selfId) ?? null;
+        const hash = userId ? avatars.get(userId) : undefined;
+        out.push({
+            channelId,
+            name: typeof label === "string" && label ? displayName(label) : (userId ?? channelId),
+            userId,
+            avatarUrl: userId && hash ? `https://cdn.discordapp.com/avatars/${userId}/${hash}.png?size=64` : null,
+            messages: messages.length,
+            files: messages.reduce((n, m) => n + m.attachments.length, 0),
+            firstAt: messages[0].sentAt,
+            lastAt: messages[messages.length - 1].sentAt,
+        });
+    }
+    return out.sort((a, b) => b.messages - a.messages);
+}
+/** One DM's messages, oldest first. */
+export async function readDiscordDm(files, channelId) {
+    const idx = indexNames(files.names);
+    const dir = channelFolder(idx, channelId);
+    return dir ? messagesIn(files, idx, dir) : [];
+}
+/** The earliest `ex` (hex Unix seconds) among signed CDN links, or null when none carry one. */
+export function linksExpireAt(urls) {
+    let soonest = null;
+    for (const u of urls) {
+        const ex = /[?&]ex=([0-9a-f]+)/i.exec(u)?.[1];
+        if (!ex)
+            continue;
+        const at = parseInt(ex, 16) * 1000;
+        if (soonest === null || at < soonest)
+            soonest = at;
+    }
+    return soonest;
+}

package/dist/index.d.ts

@@ -12,8 +12,9 @@ export { describeFolderRules, folderFollowNote, folderPhrase, } from "./permissi
 export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_URL, TERMS_VERSION, } from "./terms/termsAgreement.js";
 export { createMlsDmDriver, MlsDriverError, type MlsDriverErrorCode } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
-export { decodeMlsDmContent, encodeMlsDmContent, historyBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, type MlsDmContent, type MlsHistoryEntry, readMlsDmContent, } from "./mls/content.js";
+export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, type MlsDmContent, type MlsHistoryEntry, type MlsImportEntry, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction, type MlsReaction } from "./mls/reactions.js";
+export { type DiscordPackageDm, type DiscordPackageMessage, isDiscordPackage, linksExpireAt, listDiscordDms, type PackageFiles, readDiscordDm, } from "./import/discordPackage.js";
 export type { DmSealingMode, MlsAddOwnDeviceOptions, MlsClaimedKeyPackage, MlsDecryptedMessage, MlsDeviceRecord, MlsDeviceRef, MlsDmDriver, MlsDmDriverOptions, MlsDmEvents, MlsGroupPosition, MlsGroupRecord, MlsGroupView, MlsKeyPackageRecord, MlsListedDevice, MlsLogEntry, MlsDmDiagnostics, MlsOwnDevice, MlsOwnDeviceAdd, MlsPins, MlsRefusal, MlsReply, MlsServerCapability, MlsStateStore, MlsTransport, MlsWelcomeDelivery, } from "./mls/interfaces.js";
 export { createApproverPairing, type ApproverOptions, type ApproverPairing, type ApproverState, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, type NewDeviceOptions, type NewDevicePairing, type NewDeviceState, } from "./pairing/newDevice.js";

package/dist/index.js

@@ -14,8 +14,9 @@ export { describeFolderRules, folderFollowNote, folderPhrase, } from "./permissi
 export { agreementAt, createTermsGate, GUIDELINES_URL, TERMS_STORAGE_KEY, TERMS_URL, TERMS_VERSION, } from "./terms/termsAgreement.js";
 export { createMlsDmDriver, MlsDriverError } from "./mls/dmDriver.js";
 export { mlsPinsFromPeerPins } from "./mls/pins.js";
-export { decodeMlsDmContent, encodeMlsDmContent, historyBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, readMlsDmContent, } from "./mls/content.js";
+export { decodeMlsDmContent, encodeMlsDmContent, acceptImport, historyBatches, importBatches, MAX_HISTORY_BATCH_BYTES, MLS_DM_CONTENT_VERSION, readMlsDmContent, } from "./mls/content.js";
 export { applyMlsReaction, mlsReactionAction } from "./mls/reactions.js";
+export { isDiscordPackage, linksExpireAt, listDiscordDms, readDiscordDm, } from "./import/discordPackage.js";
 export { createApproverPairing, } from "./pairing/approver.js";
 export { createNewDevicePairing, PAIRING_APPROVAL_MS, } from "./pairing/newDevice.js";
 export { createPairingRelay, PairingRelayError } from "./pairing/relayClient.js";

package/dist/mls/content.d.ts

@@ -10,7 +10,20 @@ export interface MlsHistoryEntry {
     replyTo?: string;
     attachments?: Record<string, SealedAttachmentKey>;
 }
+/** One message from the sender's own Discord data. Who wrote it is never in here: the reader takes the MLS sender. */
+export interface MlsImportEntry {
+    id: string;
+    /** Milliseconds since the epoch, Discord's time for it. */
+    sentAt: number;
+    text: string;
+    attachments?: Record<string, SealedAttachmentKey>;
+}
 export type MlsDmContent = {
+    type: "import";
+    id: string;
+    source: "discord";
+    entries: MlsImportEntry[];
+} | {
     type: "history";
     /** For the batch itself, so a repeat is the same message. */
     id: string;
@@ -48,6 +61,20 @@ export declare function encodeMlsDmContent(content: MlsDmContent): Uint8Array;
 export declare function historyBatches(entries: MlsHistoryEntry[], newId: () => string): Extract<MlsDmContent, {
     type: "history";
 }>[];
+/** An import as batches small enough to send, oldest first, the same way `historyBatches` splits. */
+export declare function importBatches(entries: MlsImportEntry[], newId: () => string): Extract<MlsDmContent, {
+    type: "import";
+}>[];
+/**
+ * An import's entries as messages by `senderId`, the MLS sender, whatever the batch says.
+ * Dated before Discord existed or after the server got it (`at`) is dropped.
+ */
+export declare function acceptImport(content: Extract<MlsDmContent, {
+    type: "import";
+}>, { senderId, at }: {
+    senderId: string;
+    at: number;
+}): MlsHistoryEntry[];
 /** Null for anything this version can't read: a newer version, a new type, or junk. */
 export declare function decodeMlsDmContent(bytes: Uint8Array): MlsDmContent | null;
 /**

package/dist/mls/content.js

@@ -85,6 +85,9 @@ export function encodeMlsDmContent(content) {
     if (content.type === "history") {
         body = { type: content.type, id: content.id, entries: content.entries.map(historyBody) };
     }
+    else if (content.type === "import") {
+        body = { type: content.type, id: content.id, source: content.source, entries: content.entries.map(importBody) };
+    }
     else if (content.type === "message") {
         body = { type: content.type, id: content.id, text: content.text };
         if (content.replyTo !== undefined)
@@ -114,24 +117,50 @@ function historyBody(e) {
         out.attachments = e.attachments;
     return out;
 }
-/** History as batches small enough to send, oldest first. One entry too big to send on its own is left out. */
-export function historyBatches(entries, newId) {
-    const size = (batch) => new TextEncoder().encode(JSON.stringify({ v: MLS_DM_CONTENT_VERSION, type: "history", id: "x".repeat(MAX_ID), entries: batch.map(historyBody) })).length;
+function importBody(e) {
+    const out = { id: e.id, sentAt: e.sentAt, text: e.text };
+    if (e.attachments && Object.keys(e.attachments).length > 0)
+        out.attachments = e.attachments;
+    return out;
+}
+function sizedBatches(entries, body, readable, head) {
+    const size = (batch) => new TextEncoder().encode(JSON.stringify({ v: MLS_DM_CONTENT_VERSION, ...head, id: "x".repeat(MAX_ID), entries: batch.map(body) })).length;
     const out = [];
     let batch = [];
     for (const e of [...entries].sort((a, b) => a.sentAt - b.sentAt)) {
-        if (!readHistoryEntry(historyBody(e)) || size([e]) > MAX_HISTORY_BATCH_BYTES)
+        if (!readable(body(e)) || size([e]) > MAX_HISTORY_BATCH_BYTES)
             continue;
         if (batch.length === MAX_HISTORY_ENTRIES || size([...batch, e]) > MAX_HISTORY_BATCH_BYTES) {
-            out.push({ type: "history", id: newId(), entries: batch });
+            out.push(batch);
             batch = [];
         }
         batch.push(e);
     }
     if (batch.length)
-        out.push({ type: "history", id: newId(), entries: batch });
+        out.push(batch);
     return out;
 }
+/** History as batches small enough to send, oldest first. One entry too big to send on its own is left out. */
+export function historyBatches(entries, newId) {
+    return sizedBatches(entries, historyBody, (raw) => readHistoryEntry(raw) !== null, { type: "history" })
+        .map((batch) => ({ type: "history", id: newId(), entries: batch }));
+}
+/** An import as batches small enough to send, oldest first, the same way `historyBatches` splits. */
+export function importBatches(entries, newId) {
+    return sizedBatches(entries, importBody, (raw) => readImportEntry(raw) !== null, { type: "import", source: "discord" })
+        .map((batch) => ({ type: "import", id: newId(), source: "discord", entries: batch }));
+}
+/** Discord's own epoch, 2015-01-01. Nothing from Discord is older. */
+const DISCORD_EPOCH = 1_420_070_400_000;
+/**
+ * An import's entries as messages by `senderId`, the MLS sender, whatever the batch says.
+ * Dated before Discord existed or after the server got it (`at`) is dropped.
+ */
+export function acceptImport(content, { senderId, at }) {
+    return content.entries
+        .filter((e) => e.sentAt >= DISCORD_EPOCH && e.sentAt <= at)
+        .map((e) => (e.attachments ? { ...e, senderId } : { id: e.id, senderId, sentAt: e.sentAt, text: e.text }));
+}
 function readHistoryEntry(v) {
     if (!isObject(v) || !isId(v.id) || !isId(v.senderId) || !isText(v.text) || !isCount(v.sentAt))
         return null;
@@ -149,6 +178,18 @@ function readHistoryEntry(v) {
     }
     return out;
 }
+function readImportEntry(v) {
+    if (!isObject(v) || !isId(v.id) || !isText(v.text) || !isCount(v.sentAt))
+        return null;
+    const out = { id: v.id, sentAt: v.sentAt, text: v.text };
+    if (v.attachments !== undefined) {
+        const attachments = readAttachments(v.attachments);
+        if (!attachments)
+            return null;
+        out.attachments = attachments;
+    }
+    return out;
+}
 /** Null for anything this version can't read: a newer version, a new type, or junk. */
 export function decodeMlsDmContent(bytes) {
     return fromRaw(parse(bytes));
@@ -161,7 +202,7 @@ export function readMlsDmContent(bytes) {
     const raw = parse(bytes);
     return fromRaw(raw) ?? (isNewer(raw) ? "newer" : null);
 }
-const KNOWN_TYPES = new Set(["message", "edit", "delete", "reaction", "history"]);
+const KNOWN_TYPES = new Set(["message", "edit", "delete", "reaction", "history", "import"]);
 const TYPE_NAME = /^[a-z][a-z0-9_]{0,31}$/;
 /** A later version, or a type this one doesn't know that names its message the way ours do. */
 function isNewer(raw) {
@@ -189,6 +230,12 @@ function fromRaw(raw) {
         const entries = raw.entries.map(readHistoryEntry);
         return entries.every((e) => e !== null) ? { type, id, entries } : null;
     }
+    if (type === "import") {
+        if (raw.source !== "discord" || !Array.isArray(raw.entries) || raw.entries.length === 0 || raw.entries.length > MAX_HISTORY_ENTRIES)
+            return null;
+        const entries = raw.entries.map(readImportEntry);
+        return entries.every((e) => e !== null) ? { type, id, source: raw.source, entries } : null;
+    }
     if (type === "message" && isText(text)) {
         const out = { type, id, text };
         if (raw.replyTo !== undefined) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gryt/core",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "description": "Gryt's shared application logic: one implementation of the things the desktop and mobile apps both do.",
   "license": "AGPL-3.0-only",
   "repository": {