buncord-hybrid-sharding 1.0.0

package/src/Structures/Data.ts

@@ -0,0 +1,33 @@
+export function getInfo() {
+    const shardList: number[] = [];
+    const parseShardList = process.env?.SHARD_LIST?.split(',') || [];
+    parseShardList.forEach(c => shardList.push(Number(c)));
+    
+    const data: ClusterClientData = {
+        SHARD_LIST: shardList,
+        TOTAL_SHARDS: Number(process.env.TOTAL_SHARDS),
+        CLUSTER_COUNT: Number(process.env.CLUSTER_COUNT),
+        CLUSTER: Number(process.env.CLUSTER),
+        CLUSTER_MANAGER_MODE: 'process',
+        MAINTENANCE: process.env.MAINTENANCE,
+        CLUSTER_QUEUE_MODE: process.env.CLUSTER_QUEUE_MODE,
+        FIRST_SHARD_ID: shardList[0] as number,
+        LAST_SHARD_ID: shardList[shardList.length - 1] as number,
+        HEARTBEAT_INTERVAL: Number(process.env.HEARTBEAT_INTERVAL) || 10000,
+    };
+    
+    return data;
+}
+
+export interface ClusterClientData {
+    SHARD_LIST: number[];
+    TOTAL_SHARDS: number;
+    LAST_SHARD_ID: number;
+    FIRST_SHARD_ID: number;
+    CLUSTER_COUNT: number;
+    MAINTENANCE?: string;
+    CLUSTER_QUEUE_MODE?: 'auto' | string | undefined;
+    CLUSTER: number;
+    CLUSTER_MANAGER_MODE: 'process';
+    HEARTBEAT_INTERVAL: number;
+}

package/src/Structures/IPCHandler.ts

@@ -0,0 +1,154 @@
+import { Cluster } from '../Core/Cluster.js';
+import { ClusterClient } from '../Core/ClusterClient.js';
+import { ClusterManager } from '../Core/ClusterManager.js';
+import { messageType } from '../types/shared.js';
+import { makePlainError } from '../Util/Util.js';
+import { Child, ChildClient } from './Child.js';
+import { RawMessage } from './IPCMessage.js';
+import { ResolveMessage } from './PromiseHandler.js';
+
+export class ClusterHandler {
+    manager: ClusterManager;
+    cluster: Cluster;
+    ipc: Child;
+    constructor(manager: ClusterManager, cluster: Cluster, ipc: Child) {
+        this.manager = manager;
+        this.cluster = cluster;
+        this.ipc = ipc;
+    }
+
+    handleMessage(message: RawMessage) {
+        if (message._type === messageType.CLIENT_READY) {
+            this.cluster.ready = true;
+            this.cluster.emit('ready');
+            this.cluster.manager._debug('Ready', this.cluster.id);
+            return;
+        }
+        if (message._type === messageType.CLIENT_BROADCAST_REQUEST) {
+            this.cluster.manager
+                .broadcastEval(message._eval, message.options)
+                ?.then(results => {
+                    return this.ipc.send({
+                        nonce: message.nonce,
+                        _type: messageType.CLIENT_BROADCAST_RESPONSE,
+                        _result: results,
+                    });
+                })
+                .catch(err => {
+                    return this.ipc.send({
+                        nonce: message.nonce,
+                        _type: messageType.CLIENT_BROADCAST_RESPONSE,
+                        _error: makePlainError(err),
+                    });
+                });
+            return;
+        }
+        if (message._type === messageType.CLIENT_MANAGER_EVAL_REQUEST) {
+            this.cluster.manager.evalOnManager(message._eval).then(result => {
+                if (result._error) {
+                    return this.ipc.send({
+                        nonce: message.nonce,
+                        _type: messageType.CLIENT_MANAGER_EVAL_RESPONSE,
+                        _error: makePlainError(result._error),
+                    });
+                }
+                return this.ipc.send({
+                    nonce: message.nonce,
+                    _type: messageType.CLIENT_MANAGER_EVAL_RESPONSE,
+                    _result: result._result,
+                });
+            });
+            return;
+        }
+        if (message._type === messageType.CLIENT_EVAL_RESPONSE) {
+            this.cluster.manager.promise.resolve(message as ResolveMessage);
+            return;
+        }
+        if (message._type === messageType.CLIENT_RESPAWN_ALL) {
+            this.cluster.manager.respawnAll(message.options);
+            return;
+        }
+        if (message._type === messageType.CLIENT_RESPAWN) {
+            this.cluster.respawn(message.options);
+            return;
+        }
+        if (message._type === messageType.CLIENT_MAINTENANCE) {
+            this.cluster.triggerMaintenance(message.maintenance);
+            return;
+        }
+        if (message._type === messageType.CLIENT_MAINTENANCE_ALL) {
+            this.cluster.manager.triggerMaintenance(message.maintenance);
+            return;
+        }
+        if (message._type === messageType.CLIENT_SPAWN_NEXT_CLUSTER) {
+            this.cluster.manager.queue.next();
+            return;
+        }
+        if (message._type === messageType.HEARTBEAT_ACK) {
+            // Heartbeat system now handles acks internally via Redis if needed, or we just ignore here
+            return;
+        }
+        if (message._type === messageType.CUSTOM_REPLY) {
+            this.cluster.manager.promise.resolve(message as ResolveMessage);
+            return;
+        }
+        return true;
+    }
+}
+
+export class ClusterClientHandler<DiscordClient> {
+    client: ClusterClient<DiscordClient>;
+    ipc: ChildClient | null;
+    constructor(client: ClusterClient<DiscordClient>, ipc: ChildClient | null) {
+        this.client = client;
+        this.ipc = ipc;
+    }
+
+    public async handleMessage(message: ResolveMessage & { date?: number; maintenance?: string }) {
+        if (message._type === messageType.CLIENT_EVAL_REQUEST) {
+            try {
+                if (!message._eval) throw new Error('Eval Script not provided');
+                this.client._respond('eval', {
+                    _eval: message._eval,
+                    _result: await this.client._eval(message._eval),
+                    _type: messageType.CLIENT_EVAL_RESPONSE,
+                    nonce: message.nonce,
+                });
+            } catch (err) {
+                this.client._respond('eval', {
+                    _eval: message._eval,
+                    _error: makePlainError(err),
+                    _type: messageType.CLIENT_EVAL_RESPONSE,
+                    nonce: message.nonce,
+                });
+            }
+            return null;
+        }
+        if (message._type === messageType.CLIENT_MANAGER_EVAL_RESPONSE) {
+            this.client.promise.resolve({ _result: message._result, _error: message._error, nonce: message.nonce });
+            return null;
+        }
+        if (message._type === messageType.CLIENT_BROADCAST_RESPONSE) {
+            this.client.promise.resolve({ _result: message._result, _error: message._error, nonce: message.nonce });
+            return null;
+        }
+        if (message._type === messageType.HEARTBEAT) {
+            this.client.send({ _type: messageType.HEARTBEAT_ACK, date: message.date });
+            return null;
+        }
+        if (message._type === messageType.CLIENT_MAINTENANCE_DISABLE) {
+            this.client.maintenance = false;
+            this.client.triggerClusterReady();
+            return null;
+        }
+        if (message._type === messageType.CLIENT_MAINTENANCE_ENABLE) {
+            this.client.maintenance = message.maintenance || true;
+            return null;
+        }
+        if (message._type === messageType.CUSTOM_REPLY) {
+            this.client.promise.resolve(message);
+            return null;
+        }
+        return true;
+    }
+}

package/src/Structures/IPCMessage.ts

@@ -0,0 +1,101 @@
+import { generateNonce } from '../Util/Util.js';
+import { messageType } from '../types/shared.js';
+import { ClusterClient } from '../Core/ClusterClient.js';
+import { Cluster } from '../Core/Cluster.js';
+
+export interface RawMessage {
+    nonce?: string;
+    _type?: number | messageType;
+    [x: string]: any;
+}
+
+export class BaseMessage {
+    [x: string]: any;
+    nonce: string;
+    private readonly _raw: RawMessage;
+    constructor(message: RawMessage) {
+        /**
+         * Creates a Message ID for identifying it for further Usage such as on replies
+         */
+        this.nonce = message.nonce || generateNonce();
+        message.nonce = this.nonce;
+
+        /**
+         * Destructs the Message Object and initializes it on the Constructor
+         */
+        this._raw = this.destructMessage(message);
+    }
+
+    /**
+     * Destructs the Message Object and initializes it on the Constructor
+     */
+    private destructMessage(message: RawMessage) {
+        for (const [key, value] of Object.entries(message)) {
+            this[key] = value;
+        }
+        if (message.nonce) this.nonce = message.nonce;
+        this._type = message._type || messageType.CUSTOM_MESSAGE;
+        return message;
+    }
+
+    public toJSON() {
+        return this._raw;
+    }
+}
+
+export class IPCMessage extends BaseMessage {
+    raw: RawMessage;
+    instance: ClusterClient<any> | Cluster;
+    constructor(instance: ClusterClient<any> | Cluster, message: RawMessage) {
+        super(message);
+
+        /**
+         * Instance, which can be the ParentCluster or the ClusterClient
+         */
+        this.instance = instance;
+
+        /**
+         * The Base Message, which is saved on the raw field.
+         */
+        this.raw = new BaseMessage(message).toJSON();
+    }
+
+    /**
+     * Sends a message to the cluster's process/worker or to the ParentCluster.
+     */
+    public async send(message: object) {
+        if (typeof message !== 'object') throw new TypeError('The Message has to be a object');
+        const baseMessage = new BaseMessage({ ...message, _type: messageType.CUSTOM_MESSAGE });
+        return this.instance.send(baseMessage.toJSON());
+    }
+
+    /**
+     * Sends a Request to the cluster's process/worker or to the ParentCluster.
+     */
+    public async request(message: object) {
+        if (typeof message !== 'object') throw new TypeError('The Message has to be a object');
+        const baseMessage = new BaseMessage({ ...message, _type: messageType.CUSTOM_REQUEST, nonce: this.nonce });
+        return this.instance.request(baseMessage.toJSON());
+    }
+
+    /**
+     * Sends a Reply to Message from the cluster's process/worker or the ParentCluster.
+     */
+    public async reply(message: object) {
+        if (typeof message !== 'object') throw new TypeError('The Message has to be a object');
+        const baseMessage = new BaseMessage({
+            ...message,
+            _type: messageType.CUSTOM_REPLY,
+            nonce: this.nonce,
+            _result: message,
+        });
+        return this.instance.send(baseMessage.toJSON());
+    }
+
+    /**
+     * Gets the type of the Message from the raw properties
+     */
+    public getType() {
+        return this.raw._type ?? messageType.CUSTOM_MESSAGE
+    }
+}

package/src/Structures/ManagerHooks.ts

@@ -0,0 +1,9 @@
+import type { Cluster } from '../Core/Cluster';
+
+export class ClusterManagerHooks {
+    constructor() {}
+
+    constructClusterArgs(cluster: Cluster, args: string[]) {
+        return args;
+    }
+}

package/src/Structures/PromiseHandler.ts

@@ -0,0 +1,63 @@
+import { generateNonce } from '../Util/Util.js';
+import { RawMessage } from './IPCMessage.js';
+
+export interface StoredPromise {
+    resolve(value: any): void;
+    reject(error: Error): void;
+    options: PromiseCreateOptions;
+    timeout?: any;
+}
+
+export interface ResolveMessage {
+    _error: { message: string; stack: string; name: string };
+    _result: any;
+    _eval?: string;
+    _type?: number;
+    nonce: string;
+}
+
+export interface PromiseCreateOptions {
+    timeout?: number;
+}
+
+export class PromiseHandler {
+    nonce: Map<string, StoredPromise>;
+    constructor() {
+        this.nonce = new Map();
+    }
+    public resolve(message: ResolveMessage) {
+        const promise = this.nonce.get(message.nonce);
+        if (promise) {
+            if (promise.timeout) clearTimeout(promise.timeout);
+            this.nonce.delete(message.nonce);
+            if (message._error) {
+                const error = new Error(message._error.message);
+                error.stack = message._error.stack;
+                error.name = message._error.name;
+                promise.reject(error);
+            } else {
+                promise.resolve(message._result);
+            }
+        }
+    }
+
+    public async create(
+        message: RawMessage & { options?: PromiseCreateOptions; stack?: string },
+        options: PromiseCreateOptions,
+    ) {
+        if (!options) options = {};
+        if (Object.keys(options).length === 0 && message.options) options = message.options;
+        if (!message.nonce) message.nonce = generateNonce();
+        return await new Promise((resolve, reject) => {
+            if (options.timeout) {
+                const timeout = setTimeout(() => {
+                    this.nonce.delete(message.nonce as string);
+                    const error = new Error('Promise timed out');
+                    error.stack = message.stack || error.stack;
+                    reject(error);
+                }, options.timeout);
+                this.nonce.set(message.nonce as string, { resolve, reject, options, timeout });
+            } else this.nonce.set(message.nonce as string, { resolve, reject, options });
+        });
+    }
+}

package/src/Structures/Queue.ts

@@ -0,0 +1,84 @@
+import { QueueOptions } from '../types/shared.js';
+import { delayFor } from '../Util/Util.js';
+
+export interface QueueItem {
+    run(...args: any): Promise<any>;
+    args: any[];
+    time?: number;
+    timeout: number;
+}
+
+export class Queue {
+    queue: QueueItem[];
+    options: Required<QueueOptions>;
+    paused: boolean;
+    constructor(options: Required<QueueOptions>) {
+        this.options = options;
+        this.queue = [];
+        this.paused = false;
+    }
+
+    /**
+     * Starts the queue and run's the item functions
+     */
+    public async start() {
+        if (!this.options.auto) {
+            return new Promise(resolve => {
+                const interval = setInterval(() => {
+                    if (this.queue.length === 0) {
+                        clearInterval(interval);
+                        resolve('Queue finished');
+                    }
+                }, 200);
+            });
+        }
+
+        const length = this.queue.length;
+        for (let i = 0; i < length; i++) {
+            if (!this.queue[0]) continue;
+            const timeout = this.queue[0].timeout;
+            await this.next();
+            await delayFor(timeout);
+        }
+        return this;
+    }
+
+    /**
+     * Goes to the next item in the queue
+     */
+    public async next() {
+        if (this.paused) return;
+        const item = this.queue.shift();
+        if (!item) return true;
+        return item.run(...item.args);
+    }
+
+    /**
+     * Stop's the queue and blocks the next item from running
+     */
+    public stop() {
+        this.paused = true;
+        return this;
+    }
+
+    /**
+     * Resume's the queue
+     */
+    public resume() {
+        this.paused = false;
+        return this;
+    }
+
+    /**
+     * Adds an item to the queue
+     */
+    public add(item: QueueItem) {
+        this.queue.push({
+            run: item.run,
+            args: item.args,
+            time: Date.now(),
+            timeout: item.timeout ?? this.options.timeout,
+        });
+        return this;
+    }
+}

package/src/Util/RedisClient.ts

@@ -0,0 +1,77 @@
+import { createClient, type RedisClientType } from 'redis';
+
+export class RedisClient {
+    public client: RedisClientType;
+    private isConnected = false;
+
+    constructor(url: string = process.env.REDIS_URL || 'redis://localhost:6379') {
+        this.client = createClient({
+            url,
+            socket: {
+                reconnectStrategy: (retries) => {
+                    return Math.min(retries * 100, 3000);
+                },
+            },
+        });
+
+        this.setupEvents();
+    }
+
+    private setupEvents() {
+        this.client.on('connect', () => {
+            this.isConnected = true;
+        });
+
+        this.client.on('error', (err) => {
+            this.isConnected = false;
+        });
+
+        this.client.on('reconnecting', () => {
+        });
+    }
+
+    async connect(): Promise<void> {
+        if (!this.isConnected) {
+            await this.client.connect();
+        }
+    }
+
+    async set(key: string, value: any, ttl?: number): Promise<void> {
+        const serialized = JSON.stringify(value);
+
+        if (ttl) {
+            await this.client.setEx(key, ttl, serialized);
+        } else {
+            await this.client.set(key, serialized);
+        }
+    }
+
+    async get<T>(key: string): Promise<T | null> {
+        const data = await this.client.get(key);
+        return data ? JSON.parse(data) : null;
+    }
+
+    async del(key: string): Promise<void> {
+        await this.client.del(key);
+    }
+
+    async publish(channel: string, message: any): Promise<void> {
+        await this.client.publish(channel, JSON.stringify(message));
+    }
+
+    async subscribe(channel: string, callback: (message: any) => void): Promise<void> {
+        const subscriber = this.client.duplicate();
+        await subscriber.connect();
+
+        await subscriber.subscribe(channel, (message) => {
+            callback(JSON.parse(message));
+        });
+    }
+
+    async disconnect(): Promise<void> {
+        if (this.isConnected) {
+            await this.client.disconnect();
+            this.isConnected = false;
+        }
+    }
+}

package/src/Util/Util.ts

@@ -0,0 +1,62 @@
+import { DefaultOptions, Endpoints } from "../types/shared.js";
+
+export function generateNonce() {
+    return Date.now().toString(36) + Math.random().toString(36);
+}
+
+export function chunkArray(array: any[], chunkSize: number) {
+    const R = [];
+    for (let i = 0; i < array.length; i += chunkSize) R.push(array.slice(i, i + chunkSize));
+    return R;
+}
+
+export function arraysAreTheSame(firstArray: any[], secondArray: any[]) {
+    return firstArray.length === secondArray.length && firstArray.every((element, index) => element === secondArray[index]);
+}
+
+export function delayFor(ms: number) {
+    if(ms < 0) return;
+    return new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+}
+
+export function makePlainError(err: Error) {
+    return {
+        name: err['name'],
+        message: err['message'],
+        stack: err['stack'],
+    };
+}
+
+export function shardIdForGuildId(guildId: string, totalShards = 1) {
+    const shard = Number(BigInt(guildId) >> BigInt(22)) % totalShards;
+    if (shard < 0)
+        throw new Error(
+            'SHARD_MISCALCULATION_SHARDID_SMALLER_THAN_0 ' +
+                `Calculated Shard: ${shard}, guildId: ${guildId}, totalShards: ${totalShards}`,
+        );
+    return shard;
+}
+
+export async function fetchRecommendedShards(token: string, guildsPerShard = 1000) {
+    if (!token) throw new Error('DISCORD_TOKEN_MISSING');
+
+    const res = await fetch(
+        `${DefaultOptions.http.api}/v${DefaultOptions.http.version}${Endpoints.botGateway}`,
+        {
+            method: 'GET',
+            headers: {
+                Authorization: `Bot ${token.replace(/^Bot\s*/i, '')}`,
+            },
+        }
+    );
+
+    if (!res.ok) {
+        if (res.status === 401) throw new Error('DISCORD_TOKEN_INVALID');
+        throw new Error(`Failed to fetch data. Status code: ${res.status}`);
+    }
+
+    const responseData = await res.json() as any;
+    return responseData.shards * (1000 / guildsPerShard);
+}

package/src/index.ts

@@ -0,0 +1,17 @@
+export * from './Core/Cluster.js';
+export * from './Core/ClusterClient.js';
+export * from './Core/ClusterManager.js';
+export * from './Core/DashboardServer.js';
+export * from './Plugins/HeartbeatSystem.js';
+export * from './Plugins/ReCluster.js';
+export * from './Plugins/AutoResharderSystem.js';
+export * from './Plugins/QueueManager.js';
+export * from './Structures/Child.js';
+export * from './Structures/Data.js';
+export * from './Structures/IPCHandler.js';
+export * from './Structures/IPCMessage.js';
+export * from './Structures/PromiseHandler.js';
+export * from './Structures/Queue.js';
+export * from './types/shared.js';
+export * from './Util/Util.js';
+export * from './Util/RedisClient.js';