discord-message-transcript 1.2.0-dev-next.0.16 → 1.2.0-dev-next.0.19

package/dist/core/cdnResolver.js

@@ -2,48 +2,55 @@ import https from 'https';
 import http from 'http';
 import { CustomWarn } from "discord-message-transcript-base";
 import crypto from 'crypto';
+import { getCDNLimiter } from "./limiter.js";
 export async function cdnResolver(url, cdnOptions) {
-    return new Promise((resolve, reject) => {
-        const client = url.startsWith('https') ? https : http;
-        const request = client.get(url, { headers: { "User-Agent": "discord-message-transcript" } }, async (response) => {
-            if (response.statusCode !== 200) {
-                response.destroy();
-                CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of uploading to CDN.
+    const limit = getCDNLimiter();
+    return limit(async () => {
+        return new Promise((resolve, reject) => {
+            const client = url.startsWith('https') ? https : http;
+            const request = client.request(url, {
+                method: 'HEAD',
+                headers: { "User-Agent": "discord-message-transcript" }
+            }, async (response) => {
+                if (response.statusCode !== 200) {
+                    response.destroy();
+                    CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of uploading to CDN.
 Failed to fetch attachment with status code: ${response.statusCode} from ${url}.`);
-                return resolve(url);
-            }
-            const contentType = response.headers["content-type"];
-            const splitContentType = contentType ? contentType?.split('/') : [];
-            if (!contentType || splitContentType.length != 2 || splitContentType[0].length == 0 || splitContentType[1].length == 0) {
-                response.destroy();
-                CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of uploading to CDN.
+                    return resolve(url);
+                }
+                const contentType = response.headers["content-type"];
+                const splitContentType = contentType ? contentType?.split('/') : [];
+                if (!contentType || splitContentType.length != 2 || splitContentType[0].length == 0 || splitContentType[1].length == 0) {
+                    response.destroy();
+                    CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of uploading to CDN.
 Failed to receive a valid content-type from ${url}.`);
+                    return resolve(url);
+                }
+                response.destroy();
+                const isImage = contentType.startsWith('image/') && contentType !== 'image/gif';
+                const isAudio = contentType.startsWith('audio/');
+                const isVideo = contentType.startsWith('video/') || contentType === 'image/gif';
+                if ((cdnOptions.includeImage && isImage) ||
+                    (cdnOptions.includeAudio && isAudio) ||
+                    (cdnOptions.includeVideo && isVideo) ||
+                    (cdnOptions.includeOthers && !isAudio && !isImage && !isVideo)) {
+                    return resolve(await cdnRedirectType(url, contentType, cdnOptions));
+                }
                 return resolve(url);
-            }
-            response.destroy();
-            const isImage = contentType.startsWith('image/') && contentType !== 'image/gif';
-            const isAudio = contentType.startsWith('audio/');
-            const isVideo = contentType.startsWith('video/') || contentType === 'image/gif';
-            if ((cdnOptions.includeImage && isImage) ||
-                (cdnOptions.includeAudio && isAudio) ||
-                (cdnOptions.includeVideo && isVideo) ||
-                (cdnOptions.includeOthers && !isAudio && !isImage && !isVideo)) {
-                return resolve(await cdnRedirectType(url, contentType, cdnOptions));
-            }
-            return resolve(url);
-        });
-        request.on('error', (err) => {
-            CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of uploading to CDN.
+            });
+            request.on('error', (err) => {
+                CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of uploading to CDN.
 Error: ${err.message}`);
-            return resolve(url);
-        });
-        request.setTimeout(15000, () => {
-            request.destroy();
-            CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of uploading to CDN.
+                return resolve(url);
+            });
+            request.setTimeout(15000, () => {
+                request.destroy();
+                CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of uploading to CDN.
 Request timeout for ${url}.`);
-            resolve(url);
+                resolve(url);
+            });
+            request.end();
         });
-        request.end();
     });
 }
 async function cdnRedirectType(url, contentType, cdnOptions) {
@@ -61,7 +68,8 @@ Error: ${error?.message ?? error}`);
             }
         }
         case "CLOUDINARY": {
-            return '';
+            return await cloudinaryResolver(url, cdnOptions.cloudName, cdnOptions.apiKey, cdnOptions.apiSecret);
+            ;
         }
         case "UPLOADCARE": {
             return await uploadCareResolver(url, cdnOptions.publicKey);

package/dist/core/fetchMessages.d.ts

@@ -1,7 +1,8 @@
 import { TextBasedChannel } from "discord.js";
 import { JsonAuthor, JsonMessage, TranscriptOptionsBase } from "discord-message-transcript-base";
 import { CDNOptions, MapMentions } from "../types/types.js";
-export declare function fetchMessages(channel: TextBasedChannel, options: TranscriptOptionsBase, cdnOptions: CDNOptions | null, authors: Map<string, JsonAuthor>, mentions: MapMentions, after?: string): Promise<{
+export declare function fetchMessages(channel: TextBasedChannel, options: TranscriptOptionsBase, cdnOptions: CDNOptions | null, authors: Map<string, JsonAuthor>, mentions: MapMentions, before?: string): Promise<{
     messages: JsonMessage[];
     end: boolean;
+    lastMessageId?: string;
 }>;

package/dist/core/fetchMessages.js

@@ -2,8 +2,8 @@ import { EmbedType } from "discord.js";
 import { componentsToJson } from "./componentToJson.js";
 import { urlResolver } from "./urlResolver.js";
 import { getMentions } from "./getMentions.js";
-export async function fetchMessages(channel, options, cdnOptions, authors, mentions, after) {
-    const originalMessages = await channel.messages.fetch({ limit: 100, cache: false, after: after });
+export async function fetchMessages(channel, options, cdnOptions, authors, mentions, before) {
+    const originalMessages = await channel.messages.fetch({ limit: 100, cache: false, before: before });
     const rawMessages = await Promise.all(originalMessages.map(async (message) => {
         const attachments = await Promise.all(message.attachments.map(async (attachment) => {
             return {
@@ -89,9 +89,10 @@ export async function fetchMessages(channel, options, cdnOptions, authors, menti
             system: message.system,
         };
     }));
+    const lastMessageId = originalMessages.last()?.id;
     const messages = rawMessages.filter(m => !(!options.includeEmpty && m.attachments.length == 0 && m.components.length == 0 && m.content == "" && m.embeds.length == 0 && !m.poll));
     const end = originalMessages.size !== 100;
-    return { messages, end };
+    return { messages, end, lastMessageId };
 }
 function formatTimeLeftPoll(timestamp) {
     const now = new Date();

package/dist/core/imageToBase64.js

@@ -1,49 +1,53 @@
 import https from 'https';
 import http from 'http';
 import { CustomWarn } from 'discord-message-transcript-base';
+import { getBase64Limiter } from './limiter.js';
 export async function imageToBase64(url) {
-    return new Promise((resolve, reject) => {
-        const client = url.startsWith('https') ? https : http;
-        const request = client.get(url, { headers: { "User-Agent": "discord-message-transcript" } }, (response) => {
-            if (response.statusCode !== 200) {
-                response.destroy();
-                CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of converting to base64.
+    const limit = getBase64Limiter();
+    return limit(async () => {
+        return new Promise((resolve, reject) => {
+            const client = url.startsWith('https') ? https : http;
+            const request = client.get(url, { headers: { "User-Agent": "discord-message-transcript" } }, (response) => {
+                if (response.statusCode !== 200) {
+                    response.destroy();
+                    CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of converting to base64.
 Failed to fetch image with status code: ${response.statusCode} from ${url}.`);
-                return resolve(url);
-            }
-            const contentType = response.headers['content-type'];
-            if (!contentType || !contentType.startsWith('image/') || contentType === 'image/gif') {
-                response.destroy();
-                return resolve(url);
-            }
-            const chunks = [];
-            response.on('data', (chunk) => {
-                chunks.push(chunk);
-            });
-            response.on('end', () => {
-                const buffer = Buffer.concat(chunks);
-                const base64 = buffer.toString('base64');
-                resolve(`data:${contentType};base64,${base64}`);
-            });
-            response.on('error', (err) => {
-                CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of converting to base64.
+                    return resolve(url);
+                }
+                const contentType = response.headers['content-type'];
+                if (!contentType || !contentType.startsWith('image/') || contentType === 'image/gif') {
+                    response.destroy();
+                    return resolve(url);
+                }
+                const chunks = [];
+                response.on('data', (chunk) => {
+                    chunks.push(chunk);
+                });
+                response.on('end', () => {
+                    const buffer = Buffer.concat(chunks);
+                    const base64 = buffer.toString('base64');
+                    resolve(`data:${contentType};base64,${base64}`);
+                });
+                response.on('error', (err) => {
+                    CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of converting to base64.
 Stream error while fetching from ${url}.
 Error: ${err.message}`);
-                resolve(url);
+                    resolve(url);
+                });
             });
-        });
-        request.on('error', (err) => {
-            CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of converting to base64.
+            request.on('error', (err) => {
+                CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of converting to base64.
 Error fetching image from ${url}
 Error: ${err.message}`);
-            return resolve(url);
-        });
-        request.setTimeout(15000, () => {
-            request.destroy();
-            CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of converting to base64.
+                return resolve(url);
+            });
+            request.setTimeout(15000, () => {
+                request.destroy();
+                CustomWarn(`This is not an issue with the package. Using the original URL as fallback instead of converting to base64.
 Request timeout for ${url}.`);
-            resolve(url);
+                resolve(url);
+            });
+            request.end();
         });
-        request.end();
     });
 }

package/dist/core/limiter.d.ts

@@ -0,0 +1,5 @@
+export declare function getCDNLimiter(): <T>(fn: () => Promise<T>) => Promise<T>;
+export declare function getBase64Limiter(): <T>(fn: () => Promise<T>) => Promise<T>;
+export declare function setCDNConcurrency(n: number): void;
+export declare function setBase64Concurrency(n: number): void;
+export declare function createLimiter(concurrency: number): <T>(fn: () => Promise<T>) => Promise<T>;

package/dist/core/limiter.js

@@ -0,0 +1,47 @@
+const globalLimiters = {
+    cdn: createLimiter(12),
+    base64: createLimiter(6)
+};
+export function getCDNLimiter() {
+    return globalLimiters.cdn;
+}
+export function getBase64Limiter() {
+    return globalLimiters.base64;
+}
+export function setCDNConcurrency(n) {
+    globalLimiters.cdn = createLimiter(n);
+}
+export function setBase64Concurrency(n) {
+    globalLimiters.base64 = createLimiter(n);
+}
+export function createLimiter(concurrency) {
+    if (concurrency <= 0) {
+        throw new Error("Limiter must be greater than 0");
+    }
+    let active = 0;
+    const queue = [];
+    const next = () => {
+        active = Math.max(0, active - 1);
+        if (queue.length > 0 && active < concurrency) {
+            const run = queue.shift();
+            run?.();
+        }
+    };
+    return function limit(fn) {
+        return new Promise((resolve, reject) => {
+            const run = () => {
+                active++;
+                Promise.resolve()
+                    .then(fn)
+                    .then(resolve, reject)
+                    .finally(next);
+            };
+            if (active < concurrency) {
+                run();
+            }
+            else {
+                queue.push(run);
+            }
+        });
+    };
+}

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export { CreateTranscriptOptions, ConvertTranscriptOptions, TranscriptOptions, ReturnType, CDNOptions, MimeType } from "./types/types.js";
 export { ReturnFormat, LocalDate, TimeZone } from "discord-message-transcript-base";
+export { setBase64Concurrency, setCDNConcurrency } from './core/limiter.js';
 import { TextBasedChannel } from "discord.js";
 import { ConvertTranscriptOptions, CreateTranscriptOptions, OutputType, ReturnType } from "./types/types.js";
 /**

package/dist/index.js

@@ -1,5 +1,6 @@
 export { ReturnType } from "./types/types.js";
 export { ReturnFormat } from "discord-message-transcript-base";
+export { setBase64Concurrency, setCDNConcurrency } from './core/limiter.js';
 import { AttachmentBuilder } from "discord.js";
 import { Json } from "./renderers/json/json.js";
 import { fetchMessages } from "./core/fetchMessages.js";
@@ -55,9 +56,9 @@ export async function createTranscript(channel, options = {}) {
             users: new Map(),
         };
         while (true) {
-            const { messages, end } = await fetchMessages(channel, internalOptions, options.cdnOptions ?? null, authors, mentions, lastMessageID);
+            const { messages, end, lastMessageId } = await fetchMessages(channel, internalOptions, options.cdnOptions ?? null, authors, mentions, lastMessageID);
             jsonTranscript.addMessages(messages);
-            lastMessageID = messages[messages.length - 1]?.id;
+            lastMessageID = lastMessageId;
             if (end || (jsonTranscript.messages.length >= quantity && quantity != 0)) {
                 break;
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "discord-message-transcript",
-  "version": "1.2.0-dev-next.0.16",
+  "version": "1.2.0-dev-next.0.19",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -48,7 +48,7 @@
     "typescript": "^5.9.3"
   },
   "dependencies": {
-    "discord-message-transcript-base": "1.2.0-dev-next.0.16"
+    "discord-message-transcript-base": "1.2.0-dev-next.0.19"
   },
   "peerDependencies": {
     "discord.js": ">=14.19.0 <15"