@vectorstores/discord 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) vectorstores contributors 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/dist/index.cjs

@@ -0,0 +1,93 @@
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var rest = require('@discordjs/rest');
+var core = require('@vectorstores/core');
+var env = require('@vectorstores/env');
+var v10 = require('discord-api-types/v10');
+
+/**
+ * Represents a reader for Discord messages using @discordjs/rest
+ * See https://github.com/discordjs/discord.js/tree/main/packages/rest
+ */ class DiscordReader {
+    constructor(discordToken, requestHandler){
+        const token = discordToken ?? env.getEnv("DISCORD_TOKEN");
+        if (!token) {
+            throw new Error("Must specify `discordToken` or set environment variable `DISCORD_TOKEN`.");
+        }
+        const restOptions = {
+            version: "10"
+        };
+        // Use the provided request handler if specified
+        if (requestHandler) {
+            restOptions.makeRequest = requestHandler;
+        }
+        this.client = new rest.REST(restOptions).setToken(token);
+    }
+    // Read all messages in a channel given a channel ID
+    async readChannel(channelId, limit, additionalInfo, oldestFirst) {
+        const params = new URLSearchParams();
+        if (limit) params.append("limit", limit.toString());
+        if (oldestFirst) params.append("after", "0");
+        try {
+            const endpoint = `${v10.Routes.channelMessages(channelId)}?${params}`;
+            const messages = await this.client.get(endpoint);
+            return messages.map((msg)=>this.createDocumentFromMessage(msg, additionalInfo));
+        } catch (err) {
+            console.error(err);
+            return [];
+        }
+    }
+    createDocumentFromMessage(msg, additionalInfo) {
+        let content = msg.content || "";
+        // Include information from embedded messages
+        if (additionalInfo && msg.embeds.length > 0) {
+            content += "\n" + msg.embeds.map((embed)=>this.embedToString(embed)).join("\n");
+        }
+        // Include URL from attachments
+        if (additionalInfo && msg.attachments.length > 0) {
+            content += "\n" + msg.attachments.map((attachment)=>`Attachment: ${attachment.url}`).join("\n");
+        }
+        return new core.Document({
+            text: content,
+            id_: msg.id,
+            metadata: {
+                messageId: msg.id,
+                username: msg.author.username,
+                createdAt: new Date(msg.timestamp).toISOString(),
+                editedAt: msg.edited_timestamp ? new Date(msg.edited_timestamp).toISOString() : undefined
+            }
+        });
+    }
+    // Create a string representation of an embedded message
+    embedToString(embed) {
+        let result = "***Embedded Message***\n";
+        if (embed.title) result += `**${embed.title}**\n`;
+        if (embed.description) result += `${embed.description}\n`;
+        if (embed.url) result += `${embed.url}\n`;
+        if (embed.fields) {
+            result += embed.fields.map((field)=>`**${field.name}**: ${field.value}`).join("\n");
+        }
+        return result.trim();
+    }
+    /**
+   * Loads messages from multiple discord channels and returns an array of Document Objects.
+   *
+   * @param {string[]} channelIds - An array of channel IDs from which to load data.
+   * @param {number} [limit] - An optional limit on the number of messages to load per channel.
+   * @param {boolean} [additionalInfo] - An optional flag to include content from embedded messages and attachments urls as text.
+   * @param {boolean} [oldestFirst] - An optional flag to load oldest messages first.
+   * @return {Promise<Document[]>} A promise that resolves to an array of loaded documents.
+   */ async loadData(channelIds, limit, additionalInfo, oldestFirst) {
+        let results = [];
+        for (const channelId of channelIds){
+            if (typeof channelId !== "string") {
+                throw new Error(`Channel id ${channelId} must be a string.`);
+            }
+            const channelDocuments = await this.readChannel(channelId, limit, additionalInfo, oldestFirst);
+            results = results.concat(channelDocuments);
+        }
+        return results;
+    }
+}
+
+exports.DiscordReader = DiscordReader;

package/dist/index.d.ts

@@ -0,0 +1,26 @@
+import { RESTOptions } from '@discordjs/rest';
+import { BaseReader, Document } from '@vectorstores/core';
+
+/**
+ * Represents a reader for Discord messages using @discordjs/rest
+ * See https://github.com/discordjs/discord.js/tree/main/packages/rest
+ */
+declare class DiscordReader implements BaseReader {
+    private client;
+    constructor(discordToken?: string, requestHandler?: RESTOptions["makeRequest"]);
+    private readChannel;
+    private createDocumentFromMessage;
+    private embedToString;
+    /**
+     * Loads messages from multiple discord channels and returns an array of Document Objects.
+     *
+     * @param {string[]} channelIds - An array of channel IDs from which to load data.
+     * @param {number} [limit] - An optional limit on the number of messages to load per channel.
+     * @param {boolean} [additionalInfo] - An optional flag to include content from embedded messages and attachments urls as text.
+     * @param {boolean} [oldestFirst] - An optional flag to load oldest messages first.
+     * @return {Promise<Document[]>} A promise that resolves to an array of loaded documents.
+     */
+    loadData(channelIds: string[], limit?: number, additionalInfo?: boolean, oldestFirst?: boolean): Promise<Document[]>;
+}
+
+export { DiscordReader };

package/dist/index.js

@@ -0,0 +1,91 @@
+import { REST } from '@discordjs/rest';
+import { Document } from '@vectorstores/core';
+import { getEnv } from '@vectorstores/env';
+import { Routes } from 'discord-api-types/v10';
+
+/**
+ * Represents a reader for Discord messages using @discordjs/rest
+ * See https://github.com/discordjs/discord.js/tree/main/packages/rest
+ */ class DiscordReader {
+    constructor(discordToken, requestHandler){
+        const token = discordToken ?? getEnv("DISCORD_TOKEN");
+        if (!token) {
+            throw new Error("Must specify `discordToken` or set environment variable `DISCORD_TOKEN`.");
+        }
+        const restOptions = {
+            version: "10"
+        };
+        // Use the provided request handler if specified
+        if (requestHandler) {
+            restOptions.makeRequest = requestHandler;
+        }
+        this.client = new REST(restOptions).setToken(token);
+    }
+    // Read all messages in a channel given a channel ID
+    async readChannel(channelId, limit, additionalInfo, oldestFirst) {
+        const params = new URLSearchParams();
+        if (limit) params.append("limit", limit.toString());
+        if (oldestFirst) params.append("after", "0");
+        try {
+            const endpoint = `${Routes.channelMessages(channelId)}?${params}`;
+            const messages = await this.client.get(endpoint);
+            return messages.map((msg)=>this.createDocumentFromMessage(msg, additionalInfo));
+        } catch (err) {
+            console.error(err);
+            return [];
+        }
+    }
+    createDocumentFromMessage(msg, additionalInfo) {
+        let content = msg.content || "";
+        // Include information from embedded messages
+        if (additionalInfo && msg.embeds.length > 0) {
+            content += "\n" + msg.embeds.map((embed)=>this.embedToString(embed)).join("\n");
+        }
+        // Include URL from attachments
+        if (additionalInfo && msg.attachments.length > 0) {
+            content += "\n" + msg.attachments.map((attachment)=>`Attachment: ${attachment.url}`).join("\n");
+        }
+        return new Document({
+            text: content,
+            id_: msg.id,
+            metadata: {
+                messageId: msg.id,
+                username: msg.author.username,
+                createdAt: new Date(msg.timestamp).toISOString(),
+                editedAt: msg.edited_timestamp ? new Date(msg.edited_timestamp).toISOString() : undefined
+            }
+        });
+    }
+    // Create a string representation of an embedded message
+    embedToString(embed) {
+        let result = "***Embedded Message***\n";
+        if (embed.title) result += `**${embed.title}**\n`;
+        if (embed.description) result += `${embed.description}\n`;
+        if (embed.url) result += `${embed.url}\n`;
+        if (embed.fields) {
+            result += embed.fields.map((field)=>`**${field.name}**: ${field.value}`).join("\n");
+        }
+        return result.trim();
+    }
+    /**
+   * Loads messages from multiple discord channels and returns an array of Document Objects.
+   *
+   * @param {string[]} channelIds - An array of channel IDs from which to load data.
+   * @param {number} [limit] - An optional limit on the number of messages to load per channel.
+   * @param {boolean} [additionalInfo] - An optional flag to include content from embedded messages and attachments urls as text.
+   * @param {boolean} [oldestFirst] - An optional flag to load oldest messages first.
+   * @return {Promise<Document[]>} A promise that resolves to an array of loaded documents.
+   */ async loadData(channelIds, limit, additionalInfo, oldestFirst) {
+        let results = [];
+        for (const channelId of channelIds){
+            if (typeof channelId !== "string") {
+                throw new Error(`Channel id ${channelId} must be a string.`);
+            }
+            const channelDocuments = await this.readChannel(channelId, limit, additionalInfo, oldestFirst);
+            results = results.concat(channelDocuments);
+        }
+        return results;
+    }
+}
+
+export { DiscordReader };

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@vectorstores/discord",
+  "description": "Discord Reader for vectorstores",
+  "version": "0.1.0",
+  "type": "module",
+  "types": "dist/index.d.ts",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/schiesser/vectorstores.git",
+    "directory": "packages/providers/discord"
+  },
+  "devDependencies": {
+    "@vectorstores/core": "0.1.0",
+    "@vectorstores/env": "0.1.0"
+  },
+  "peerDependencies": {
+    "@vectorstores/core": "0.1.0",
+    "@vectorstores/env": "0.1.0"
+  },
+  "dependencies": {
+    "@discordjs/rest": "^2.3.0",
+    "discord-api-types": "^0.37.105"
+  },
+  "scripts": {
+    "build": "bunchee",
+    "dev": "bunchee --watch"
+  }
+}