discord-protos 1.0.4 → 1.2.43

package/dist/load.js

@@ -3,17 +3,153 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const puppeteer_1 = require("puppeteer");
 const fs_1 = require("fs");
 const path_1 = require("path");
-const script = (0, fs_1.readFileSync)((0, path_1.join)(__dirname, "parse.js"), "utf8");
+const child_process_1 = require("child_process");
+// For the JS template, all we need to do is update the exports at the bottom
+const JS_TEMPLATE = `import { MessageType } from "@protobuf-ts/runtime";
+
+/**
+ * Supports both node and web environments, replacement of previous Buffer.from() being node-only.
+ * This is specific to this package's usage, and not a replacement of Buffer.from() altogether
+ */
+const compatBuffer = {
+    from: function (input: string | Uint8Array, encoding?: string) {
+        if (typeof input === "string" && encoding === "base64") {
+            const encodedBytes = atob(input);
+            const bytes = new Uint8Array(encodedBytes.length);
+            for (let i = 0; i < encodedBytes.length; i++) {
+                bytes[i] = encodedBytes.charCodeAt(i);
+            }
+            return bytes;
+        } else if (!encoding && input instanceof Uint8Array) {
+            return input;
+        }
+        throw new Error("Invalid input type.");
+    },
+    toBase64String: function (buffer: Uint8Array) {
+        let encodedBytes = "";
+        for (let i = 0; i < buffer.length; i++) {
+            encodedBytes += String.fromCharCode(buffer[i]);
+        }
+        return btoa(encodedBytes);
+    },
+};
+
+function toBase64(this: MessageType<any>, data) {
+    return compatBuffer.toBase64String(compatBuffer.from(this.toBinary(data)));
+}
+
+function fromBase64(this: MessageType<any>, base64: string) {
+    return this.fromBinary(compatBuffer.from(base64, "base64"));
+}
+
+declare module "@protobuf-ts/runtime" {
+    interface MessageType<T> {
+        toBase64(data: T): string;
+        fromBase64(base64: string): T;
+    }
+}
+
+MessageType.prototype.fromBase64 = fromBase64;
+MessageType.prototype.toBase64 = toBase64;
+
+{{ protos_exports }}
+`;
+// For the Python template, we need to update both imports and the __all__ variable, as well as bump the version number
+const PY_TEMPLATE = `from __future__ import annotations
+
+from enum import Enum as _Enum
+from typing import TYPE_CHECKING
+
+__version__ = '{{ version }}'
+
+if TYPE_CHECKING:
+    from google.protobuf.message import Message as _Message
+
+    {{ protos_equals }} = _Message
+else:
+{{ protos_imports }}
+
+__all__ = (
+    '__version__',
+    'UserSettingsType',
+{{ exports }}
+)
+
+
+class UserSettingsType(_Enum):
+    preloaded_user_settings = 1
+    frecency_user_settings = 2
+    test_settings = 3
+
+
+UserSettingsImpl = {
+    UserSettingsType.preloaded_user_settings: PreloadedUserSettings,
+    UserSettingsType.frecency_user_settings: FrecencyUserSettings,
+    UserSettingsType.test_settings: None,
+}
+`;
+// This must be run early
+const PRELOAD_SCRIPT = (0, fs_1.readFileSync)((0, path_1.join)(__dirname, "..", "scripts", "preload.js"), "utf8");
+const PARSE_SCRIPT = (0, fs_1.readFileSync)((0, path_1.join)(__dirname, "..", "scripts", "parse.js"), "utf8");
 async function main() {
-    const browser = await puppeteer_1.default.launch({
-        headless: true,
-    });
+    const browser = await puppeteer_1.default.launch();
     const page = await browser.newPage();
-    await page.goto("https://discord.com/app", { waitUntil: "networkidle0" });
-    const protos = await page.evaluate(`${script}; protos`);
+    page.on("console", (msg) => console.debug(msg.text()));
+    // Preload script grabs the objects
+    await page.evaluateOnNewDocument(PRELOAD_SCRIPT);
+    await page.goto("https://canary.discord.com/app", { waitUntil: "networkidle0" });
+    const protos = await page.evaluate(`${PARSE_SCRIPT}; protos`);
+    await browser.close();
+    // Delete all existing files and folders in the discord_protos directory
+    const outputs = [(0, path_1.join)(__dirname, "..", "discord_protos"), (0, path_1.join)(__dirname, "..", "src", "discord_protos")];
+    for (const output of outputs) {
+        if ((0, fs_1.existsSync)(output)) {
+            for (const file of (0, fs_1.readdirSync)(output)) {
+                try {
+                    if ((0, fs_1.readdirSync)((0, path_1.join)(output, file)).length > 0) {
+                        (0, fs_1.rmSync)((0, path_1.join)(output, file), { recursive: true, force: true });
+                    }
+                }
+                catch { }
+            }
+        }
+    }
+    // Write the protos to disk
+    const filenames = [];
     for (const [name, proto] of Object.entries(protos)) {
-        (0, fs_1.writeFileSync)((0, path_1.join)(__dirname, "..", "out", name + ".proto"), proto.data);
+        const dir = (0, path_1.join)(__dirname, "..", ...proto.package.split("."));
+        const filename = (0, path_1.join)(dir, `${name}.proto`);
+        filenames.push(filename.replace((0, path_1.join)(__dirname, ".."), ".").replaceAll("\\", "/"));
+        // Ensure the directory exists
+        if (!(0, fs_1.existsSync)(dir)) {
+            (0, fs_1.mkdirSync)(dir, { recursive: true });
+        }
+        (0, fs_1.writeFileSync)(filename, proto.data);
     }
-    await browser.close();
+    // Check if we have any changes using git
+    const changes = (0, child_process_1.execSync)("git status --porcelain").toString().trim();
+    if (!changes.includes(".proto")) {
+        console.log("No changes detected, exiting...");
+        return;
+    }
+    const packageJson = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)(__dirname, "..", "package.json"), "utf8"));
+    // Bump the version number
+    const version = packageJson.version.split(".").map((x) => parseInt(x)).map((x, i) => i === 2 ? x + 1 : x).join(".");
+    packageJson.version = version;
+    // This is very cursed
+    // For protoc to parse any new protos, we have to edit the script in the package.json to include the filenames
+    const js = packageJson.scripts.js.split(" ").filter((x) => !x.endsWith(".proto")).join(" ");
+    const py = packageJson.scripts.py.split(" ").filter((x) => !x.endsWith(".proto")).join(" ");
+    packageJson.scripts.js = `${js} ${filenames.join(" ")}`;
+    packageJson.scripts.py = `${py} ${filenames.join(" ")}`;
+    (0, fs_1.writeFileSync)((0, path_1.join)(__dirname, "..", "package.json"), JSON.stringify(packageJson, null, 4));
+    // Update the JS template
+    const jsExports = filenames.map((x) => `export * from "${x.replaceAll("\\", "/").replace(".proto", "")}";`).join("\n");
+    (0, fs_1.writeFileSync)((0, path_1.join)(__dirname, "..", "src", "index.ts"), JS_TEMPLATE.replace("{{ protos_exports }}", jsExports));
+    // Update the Python template
+    const pyExports = Object.keys(protos).map((x) => `    '${x}',`).join("\n");
+    const pyImports = filenames.map((x) => `    from ${x.replaceAll("\\", "/").replace("./discord_protos", "").replaceAll("/", ".").replace(".proto", "_pb2")} import *`).join("\n");
+    const pyEquals = Object.keys(protos).join(" = ");
+    (0, fs_1.writeFileSync)((0, path_1.join)(__dirname, "..", "discord_protos", "__init__.py"), PY_TEMPLATE.replace("{{ version }}", version).replace("{{ exports }}", pyExports).replace("{{ protos_imports }}", pyImports).replace("{{ protos_equals }}", pyEquals));
 }
 main();

package/package.json

@@ -1,15 +1,14 @@
 {
     "name": "discord-protos",
-    "version": "1.0.4",
+    "version": "1.2.43",
     "description": "A parser for Discord's protobufs",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
-    "packageManager": "pnpm@7.14.2",
     "contributors": [
         {
             "name": "dolfies",
-            "email": "jeyalfie47@gmail.com",
-            "url": "https://github.com/dolfies"
+            "email": "me@dolfi.es",
+            "url": "https://dolfi.es"
         },
         {
             "name": "Samuel Scheit",
@@ -17,21 +16,23 @@
             "url": "https://samuelscheit.com"
         }
     ],
+    "repository": {
+        "url": "https://github.com/discord-userdoccers/discord-protos"
+    },
     "license": "MIT",
     "scripts": {
-        "py": "protoc --python_out=./ ./out/*.proto",
-        "js": "protoc --proto_path=./out/ --ts_out src/proto PreloadedUserSettings.proto FrecencyUserSettings.proto",
+        "py": "protoc --proto_path=./discord_protos/ --python_out=discord_protos ./discord_protos/discord_users/v1/PreloadedUserSettings.proto ./discord_protos/discord_users/v1/FrecencyUserSettings.proto ./discord_protos/discord_kkv_store_value_models/v1/ApplicationUserRoleConnection.proto ./discord_protos/discord_kkv_store_value_models/v1/AcknowledgedApplicationDisclosures.proto ./discord_protos/premium_marketing/v1/PremiumMarketingComponentProperties.proto",
+        "js": "protoc --proto_path=./discord_protos/ --ts_out src/discord_protos ./discord_protos/discord_users/v1/PreloadedUserSettings.proto ./discord_protos/discord_users/v1/FrecencyUserSettings.proto ./discord_protos/discord_kkv_store_value_models/v1/ApplicationUserRoleConnection.proto ./discord_protos/discord_kkv_store_value_models/v1/AcknowledgedApplicationDisclosures.proto ./discord_protos/premium_marketing/v1/PremiumMarketingComponentProperties.proto",
         "build": "tsc",
-        "load": "yarn build && node dist/load.js",
-        "test": "yarn build && node dist/test.js"
+        "load": "npm run build && node dist/load.js"
     },
     "devDependencies": {
-        "@protobuf-ts/plugin": "^2.8.2",
-        "@protobuf-ts/protoc": "^2.8.2",
-        "puppeteer": "^19.7.0",
-        "typescript": "^4.9.5"
+        "@protobuf-ts/plugin": "^2.9.6",
+        "@protobuf-ts/protoc": "^2.9.6",
+        "puppeteer": "^24.6.0",
+        "typescript": "^5.8.2"
     },
     "dependencies": {
-        "@protobuf-ts/runtime": "^2.8.2"
+        "@protobuf-ts/runtime": "^2.9.6"
     }
 }

package/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 125
+skip-string-normalization = true
+
+[tool.pyright]
+include = ["discord_protos"]
+exclude = [
+    "**/__pycache__",
+    "src",
+    "node_modules",
+]
+reportUnnecessaryTypeIgnoreComment = "warning"
+reportUnusedImport = "error"
+reportShadowedImports = false
+pythonVersion = "3.8"
+typeCheckingMode = "basic"
+useLibraryCodeForTypes = true

package/requirements.txt

@@ -0,0 +1 @@
+protobuf

package/scripts/parse.js

@@ -0,0 +1,269 @@
+// Map the type ints to their names
+const ScalarType = {
+    1: "double",
+    2: "float",
+    3: "int64",
+    4: "uint64",
+    5: "int32",
+    6: "fixed64",
+    7: "fixed32",
+    8: "bool",
+    9: "string",
+    12: "bytes",
+    13: "uint32",
+    15: "sfixed32",
+    16: "sfixed64",
+    17: "sint32",
+    18: "sint64",
+};
+const RepeatType = {
+    NO: 0,
+    PACKED: 1,
+    UNPACKED: 2,
+};
+
+function parseType(field) {
+    // We extract the actual field if possible
+    if (typeof field === "function") {
+        field = field();
+        // If it's a real type, we just return it
+    } else if (typeof field === "number") {
+        return [ScalarType[field], []];
+    }
+
+    var type,
+        structs = [];
+
+    // The kind gives us clues on how to find the type
+    switch (field.kind) {
+        case "message":
+            type = field.T().typeName;
+            if (type.startsWith("discord")) {
+                [, type] = parseName(type);
+            }
+            break;
+        case "scalar":
+            type = ScalarType[field.T];
+            break;
+        case "map":
+            type = `map<${parseType(field.K)[0]}, ${parseType(field.V)[0]}>`;
+            break;
+        case "enum":
+            [, type] = parseName(field.T()[0]);
+            break;
+        default:
+            throw new Error(`Unknown field type: ${field?.kind || field}`);
+    }
+
+    // Now we lazily discover any protos in the fields
+    for (let t of [field.T, field.K, field.V]) {
+        t = t?.T || t;
+
+        if (typeof t === "function" && (!t().typeName || t().typeName.startsWith("discord_protos"))) {
+            t = t();
+            if (Array.isArray(t)) {
+                structs.push(parseEnum(t));
+            } else {
+                const extraStruct = parseProto(t);
+                structs.push(...(extraStruct.structs || []));
+                delete extraStruct.structs;
+                structs.push(extraStruct);
+            }
+        }
+    }
+
+    return [type, structs];
+}
+
+function parseName(name) {
+    const split = name.split(".");
+    return [split.slice(0, -1).join("."), split.slice(-1)[0]];
+}
+
+function convertCase(str) {
+    // This converts standard EnumType case to ENUM_TYPE
+    // it must support the special case SlayerSDKReceive to SLAYER_SDK_RECEIVE
+    return str
+        // HACK: I can't think of another way to fix this
+        .replace("DMs", "Dms")
+        .replace(/([a-z])([A-Z])/g, "$1_$2")
+        .replace(/([A-Z])([A-Z][a-z])/g, "$1_$2")
+        .replace(/([a-z])(\d)/g, "$1_$2")
+        .replace(/([A-Z]+)(?=[A-Z][a-z])/g, "$1_")
+        .toUpperCase();
+}
+
+function camelToSnake(str) {
+    return str.replace(/(([a-z])(?=[A-Z][a-zA-Z])|([A-Z])(?=[A-Z][a-z]))/g,'$1_').toLowerCase()
+}
+
+function flattenField(field) {
+    const [type, structs] = parseType(field);
+    return [
+        {
+            number: field.no,
+            name: field.name,
+            kind: field.kind,
+            type: type,
+            oneof: field.oneof,
+            optional: field.opt,
+            repeated: Boolean(field.repeat),
+            unpacked: field.repeat === RepeatType.UNPACKED,
+        },
+        structs,
+    ];
+}
+
+function parseEnum(enun) {
+    const [name, data] = enun;
+
+    // Protobuf enum values conventionally have a prefix of NAME_ and we must add that back
+    const [, formattedName] = parseName(name);
+    const prefix = convertCase(formattedName) + "_";
+
+    return {
+        name: formattedName,
+        kind: "enum",
+        values: Object.entries(data)
+            .filter(([k, _]) => isNaN(Number(k)))
+            .map(([k, v]) => ({
+                // Discord shitcode sometimes fails to strip the prefix on the special case
+                // so we check if it already has it
+                name: k.startsWith(prefix) ? k : prefix + k,
+                value: v,
+            })),
+    };
+}
+
+function parseProto(proto) {
+    const fields = [];
+    const structs = [];
+    proto.fields.forEach(function (field) {
+        const [f, s] = flattenField(field);
+        fields.push(f);
+        structs.push(...s);
+    });
+
+    const seen = new Set();
+    const [package, name] = parseName(proto.typeName);
+    return {
+        package,
+        name,
+        kind: "message",
+        fields: fields,
+        structs: structs.filter((v) => (seen.has(v.name) ? false : seen.add(v.name))),
+    };
+}
+
+function extractProtos() {
+    const results = {};
+
+    // Populated by preload.js
+    for (const proto of window.protoObjects) {
+        if (!proto?.typeName?.startsWith?.("discord_protos")) continue;
+        const [, name] = parseName(proto.typeName);
+        console.log(`Parsing ${name}...`);
+        results[name] = parseProto(proto);
+    }
+
+    // Remove every proto that is mentioned in another proto
+    for (const proto of Object.values(results)) {
+        for (const struct of proto.structs) {
+            delete results[struct.name];
+        }
+    }
+
+    return results;
+}
+
+function createProtoField(field) {
+    if (!field.repeated && field.unpacked)
+        throw new Error(`Field ${field.name} is not repeated but has unpacked set`);
+    return `${field.optional ? "optional " : field.repeated ? "repeated " : ""}${field.type} ${field.name} = ${field.number}${field.unpacked ? " [packed = false]" : ""};`;
+}
+
+function createProtoFile(proto) {
+    const lines = [`syntax = "proto3";\n`, `package ${proto.package};\n`, `message ${proto.name} {`];
+
+    proto.structs.forEach((struct) => {
+        lines.push(`  ${struct.kind} ${struct.name} {`);
+
+        switch (struct.kind) {
+            case "enum":
+                struct.values.forEach((value) => {
+                    lines.push(`    ${value.name.toUpperCase()} = ${value.value};`);
+                });
+                break;
+            case "message":
+                // Group fields by oneof if applicable
+                const oneofGroups = new Map();
+                const normalFields = [];
+                
+                struct.fields.forEach((field) => {
+                    if (field.oneof) {
+                        if (!oneofGroups.has(field.oneof)) {
+                            oneofGroups.set(field.oneof, []);
+                        }
+                        oneofGroups.get(field.oneof).push(field);
+                    } else {
+                        normalFields.push(field);
+                    }
+                });
+
+                oneofGroups.forEach((fields, oneofName) => {
+                    lines.push(`    oneof ${camelToSnake(oneofName)} {`);
+                    fields.forEach((field) => {
+                        lines.push(`      ${createProtoField(field)}`);
+                    });
+                    lines.push(`    }`);
+                });
+
+                normalFields.forEach((field) => {
+                    lines.push(`    ${createProtoField(field)}`);
+                });
+                break;
+            default:
+                throw new Error(`Unknown struct kind: ${struct.kind}`);
+        }
+
+        lines.push(`  }\n`);
+    });
+
+    const oneofGroups = new Map();
+
+    proto.fields.forEach((field) => {
+        if (field.oneof) {
+            if (!oneofGroups.has(field.oneof)) {
+                oneofGroups.set(field.oneof, []);
+            }
+            oneofGroups.get(field.oneof).push(createProtoField(field));
+        } else {
+            lines.push(`  ${createProtoField(field)}`);
+        }
+    });
+
+    oneofGroups.forEach((fields, oneofName) => {
+        lines.push(`  oneof ${camelToSnake(oneofName)} {`);
+        fields.forEach(field => lines.push(`    ${field}`));
+        lines.push(`  }\n`);
+    });
+
+    // Check if we're using the funny Google well-knowns and insert an import statement (I love Discord)
+    if (lines.some((line) => line.includes("google.protobuf"))) {
+        lines.splice(1, 0, `import "google/protobuf/wrappers.proto";\nimport "google/protobuf/timestamp.proto";\n`);
+    }
+
+    lines.push("}\n");
+    return lines.join("\n");
+}
+
+const protos = extractProtos();
+for (const [key, proto] of Object.entries(protos)) {
+    const data = createProtoFile(proto);
+    protos[key].data = data;
+    if (window.DiscordNative?.fileManager) {
+        window.DiscordNative.fileManager.saveWithDialog(data, `${proto.name}.proto`);
+    } else {
+        console.log(data);
+    }
+}

package/scripts/preload.js

@@ -0,0 +1,17 @@
+window.protoObjects = [];
+Object.defineProperty(Object.prototype, "typeName", {
+    configurable: true,
+
+    set(v) {
+        if (this.internalBinaryRead != null) {
+            window.protoObjects.push(this);
+        }
+
+        Object.defineProperty(this, "typeName", {
+            value: v,
+            configurable: true,
+            enumerable: true,
+            writable: true
+        });
+    }
+});

package/setup.py

@@ -0,0 +1,42 @@
+from setuptools import setup
+import re
+
+with open('requirements.txt') as f:
+    requirements = f.read().splitlines()
+
+with open('discord_protos/__init__.py') as f:
+    version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', f.read(), re.MULTILINE).group(1)
+
+with open('README.md') as f:
+    readme = f.read()
+
+setup(
+    name='discord-protos',
+    author='Dolfies',
+    url='https://github.com/dolfies/discord-protos',
+    project_urls={
+        'Issue tracker': 'https://github.com/dolfies/discord-protos/issues',
+    },
+    version=version,
+    packages=['discord_protos'],
+    license='MIT',
+    description='Discord user settings protobufs.',
+    long_description=readme,
+    long_description_content_type='text/markdown',
+    include_package_data=True,
+    install_requires=requirements,
+    python_requires='>=3.8.0',
+    classifiers=[
+        'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Topic :: Internet',
+        'Topic :: Software Development :: Libraries',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Topic :: Utilities',
+    ],
+)

package/tsconfig.json

@@ -1,10 +1,12 @@
 {
     "compilerOptions": {
-        "target": "ES2020",
+        "target": "ESNext",
         "module": "commonjs",
         "allowJs": true,
         "declaration": true,
-		"outDir": "./dist",
-		"rootDir": "./src",
-    }
+        "strictNullChecks": true,
+        "outDir": "./dist",
+        "rootDir": "./src"
+    },
+    "include": ["src/*.ts", "src/*/*.ts"],
 }

package/dist/parse.d.ts

@@ -1,35 +0,0 @@
-declare function parseType(field: any): any;
-declare function parseName(name: any): any;
-declare function flattenField(field: any): any[];
-declare function parseEnum(enun: any): {
-    name: any;
-    kind: string;
-    values: {
-        name: string;
-        value: any;
-    }[];
-};
-declare function parseProto(proto: any): {
-    name: any;
-    kind: string;
-    fields: any[];
-    structs: any[];
-};
-declare function extractProtos(): {};
-declare function createProtoField(field: any): string;
-declare function createProtoFile(proto: any): string;
-declare function getModules(str: any): any;
-declare function filterMap(arr: any, callback: any): any;
-declare const REAL_TYPES: {
-    1: string;
-    2: string;
-    3: string;
-    4: string;
-    5: string;
-    6: string;
-    8: string;
-    9: string;
-    12: string;
-    13: string;
-};
-declare const protos: {};