@yrpri/api 9.0.90 → 9.0.92

package/app.js

@@ -108,9 +108,8 @@ process.on("unhandledRejection", (reason, promise) => {
     }
 });
 import { botsWithJavascript, isBadBot, } from "./bot_control.js";
-import { WebSocketServer } from "ws";
-import { v4 as uuidv4 } from "uuid";
 import { Op } from "sequelize";
+import { WebSocketsManager } from "./webSockets.js";
 export class YourPrioritiesApi {
     constructor(port = undefined) {
         this.determineVersion = (req) => {
@@ -581,7 +580,6 @@ export class YourPrioritiesApi {
         this.app.use((req, res, next) => {
             const baseDir = path.join(__dirname, "../webAppsDist");
             const useNewVersion = req.useNewVersion;
-            console.log(`------XY-----------------------> Using new version: ${useNewVersion}`);
             // Set the paths depending on the version
             req.adminAppPath = useNewVersion
                 ? path.join(baseDir, "client/dist")
@@ -592,7 +590,7 @@ export class YourPrioritiesApi {
             const staticPath = req.path.startsWith("/admin")
                 ? req.adminAppPath
                 : req.clientAppPath;
-            console.log("Static path", staticPath);
+            //console.log("Static path", staticPath);
             // Check if the request is for index.html
             if (req.path === "/" || req.path === "/index.html") {
                 index(req, res, next); // Use your dynamic handler
@@ -922,132 +920,8 @@ export class YourPrioritiesApi {
     }
     async listen() {
         const server = await this.setupHttpsServer();
-        const [pub, sub] = await this.setupPubSub();
-        this.ws = new WebSocketServer({ server: server });
-        this.ws.on("connection", (ws) => {
-            const clientId = uuidv4();
-            this.wsClients.set(clientId, ws);
-            console.log(`New WebSocket connection: clientId ${clientId}`);
-            ws.send(JSON.stringify({ clientId: clientId }));
-            ws.on("message", (message) => {
-                let parsedMessage;
-                try {
-                    parsedMessage = JSON.parse(message);
-                }
-                catch (e) {
-                    console.log(`Received non-JSON message from client ${clientId}:`, message);
-                    parsedMessage = message;
-                }
-                if (parsedMessage && parsedMessage.type === "heartbeat") {
-                    console.log(`Received heartbeat from client ${clientId}`);
-                    ws.send(JSON.stringify({ type: "heartbeat_ack" }));
-                }
-                else if (!this.wsClients.has(clientId)) {
-                    const messageToSend = JSON.stringify({
-                        clientId,
-                        action: "directMessage",
-                        data: parsedMessage,
-                    });
-                    pub
-                        .publish("ypWebsocketChannel", messageToSend)
-                        .then((reply) => {
-                        console.log(`Message published to ypWebsocketChannel: ${reply}`);
-                    })
-                        .catch((err) => {
-                        console.error(`Error publishing to Redis channel ypWebsocketChannel:`, err);
-                    });
-                }
-            });
-            ws.on("close", () => {
-                this.wsClients.delete(clientId);
-                console.log(`WebSocket connection closed: clientId ${clientId}`);
-            });
-            ws.on("error", (err) => {
-                this.wsClients.delete(clientId);
-                console.error(`WebSocket error with clientId ${clientId}:`, err);
-            });
-        });
-    }
-    async setupPubSub() {
-        const pub = this.redisClient.duplicate();
-        const sub = this.redisClient.duplicate();
-        pub.on("error", (err) => {
-            console.error("Publisher Redis client error:", err);
-        });
-        pub.on("connect", () => {
-            console.log("Publisher Redis client is connected");
-        });
-        pub.on("reconnecting", () => {
-            console.log("Publisher Redis client is reconnecting");
-        });
-        sub.on("error", (err) => {
-            console.error("Subscriber Redis client error:", err);
-        });
-        sub.on("connect", () => {
-            console.log("Subscriber Redis client is connected");
-        });
-        sub.on("reconnecting", () => {
-            console.log("Subscriber Redis client is reconnecting");
-        });
-        try {
-            await Promise.all([pub.connect(), sub.connect()]);
-        }
-        catch (err) {
-            console.error("Error connecting to Redis:", err);
-        }
-        sub.subscribe("ypWebsocketChannel", (message, channel) => {
-            try {
-                const parsedMessage = JSON.parse(message);
-                const { clientId, action, data } = parsedMessage;
-                console.log(`Received message from Redis: ${message} at ${channel}`);
-                switch (action) {
-                    case "broadcast":
-                        this.wsClients.forEach((ws, id) => {
-                            try {
-                                ws.send(JSON.stringify(data));
-                            }
-                            catch (err) {
-                                console.error(`Error sending broadcast message to client ${id}:`, err);
-                            }
-                        });
-                        break;
-                    case "directMessage":
-                        const ws = this.wsClients.get(clientId);
-                        if (ws) {
-                            try {
-                                ws.send(JSON.stringify(data));
-                            }
-                            catch (err) {
-                                console.error(`Error sending direct message to client ${clientId}:`, err);
-                            }
-                        }
-                        else {
-                            console.warn(`No WebSocket found for clientId ${clientId}`);
-                            this.wsClients.delete(clientId);
-                        }
-                        break;
-                }
-            }
-            catch (e) {
-                console.error("Error handling message from Redis:", e);
-            }
-        });
-        return [pub, sub];
-    }
-    handleLocalMessage(clientId, parsedMessage) {
-        const ws = this.wsClients.get(clientId);
-        if (ws) {
-            try {
-                ws.send(parsedMessage);
-            }
-            catch (err) {
-                console.error(`Error sending message to client ${clientId}:`, err);
-            }
-        }
-        else {
-            console.warn(`No WebSocket found for clientId ${clientId}`);
-            this.wsClients.delete(clientId);
-        }
+        this.webSocketsManager = new WebSocketsManager(this.wsClients, this.redisClient, server);
+        await this.webSocketsManager.listen();
     }
     setupHttpsServer() {
         let server;

package/controllers/images.cjs

@@ -222,26 +222,71 @@ router.post("/:imageId/comment", auth.isLoggedInNoAnonymousCheck, auth.can("view
 router.post("/", isAuthenticated, async function (req, res) {
     try {
         const s3 = new aws.S3();
-        //TODO: Look into making animated gifs work through sharp
-        const isGif = req.file && req.file.originalname.toLowerCase().indexOf(".gif");
+        // 1) Check if the file name ends with ".gif"
+        const isGifFilename = (filename) => {
+            const lowerCaseFilename = filename.toLowerCase();
+            console.log("filename===========>", filename);
+            console.log("lowerCaseFilename===========>", lowerCaseFilename);
+            return lowerCaseFilename.endsWith(".gif");
+        };
+        // 2) Create the storage with a Key callback that uses 'isGifFilename'
         const storage = s3Storage({
             Key: (req, file, cb) => {
                 crypto.pseudoRandomBytes(16, (err, raw) => {
-                    cb(err, err ? undefined : `${raw.toString("hex")}.${isGif ? "gif" : "png"}`);
+                    if (err)
+                        return cb(err);
+                    const rawHex = raw.toString("hex");
+                    console.log("file.originalname", file.originalname);
+                    const isGif = isGifFilename(file.originalname);
+                    console.log("gif===========>", isGif);
+                    file.outputFormat = isGif ? "gif" : "png";
+                    cb(null, `${rawHex}.${isGif ? "gif" : "png"}`);
                 });
             },
             s3,
             Bucket: process.env.S3_BUCKET,
             multiple: true,
             resize: models.Image.getSharpVersions(req.query.itemType),
-            toFormat: isGif ? "gif" : "png",
+            toFormat: "png",
         });
-        const upload = multer({ storage });
+        // 3) Allowed MIME types that Sharp commonly supports
+        //    (Adjust or expand as needed)
+        const allowedMimeTypes = [
+            "image/png",
+            "image/jpeg",
+            "image/jpg",
+            "image/gif",
+            "image/webp",
+            "image/tiff",
+            "image/svg+xml"
+        ];
+        // 4) A fileFilter to reject non-image or invalid Sharp formats
+        const fileFilter = (req, file, cb) => {
+            console.log("file------------>", file);
+            // Check that it's an image and specifically in our allowed list
+            if (!allowedMimeTypes.includes(file.mimetype.toLowerCase())) {
+                return cb(new Error(`Unsupported file type: ${file.mimetype}. Allowed: ${allowedMimeTypes.join(", ")}`));
+            }
+            cb(null, true);
+        };
+        // 5) Construct multer with:
+        //    - the custom S3-based storage
+        //    - our fileFilter for validation
+        //    - a file size limit of 50MB (you can adjust as needed)
+        const upload = multer({
+            storage,
+            fileFilter,
+            limits: { fileSize: 50 * 1024 * 1024 } // 50MB limit
+        });
+        // 6) Use upload.single, handle the success/error callback carefully
         upload.single("file")(req, res, async function (error) {
             if (error) {
-                sendError(res, req.file ? req.file.originalname : "unknown filename", "create", res.user, error);
+                // Multer will throw if file is too large or invalid
+                console.error("File upload error:", error);
+                return res.status(400).json({ error: error.message });
             }
-            else {
+            // Continue if there's a valid image file
+            try {
                 const formats = JSON.stringify(models.Image.createFormatsFromSharpFile(req.file));
                 const image = models.Image.build({
                     user_id: req.user.id,
@@ -257,12 +302,21 @@ router.post("/", isAuthenticated, async function (req, res) {
                     context: "create",
                     userId: req.user ? req.user.id : -1,
                 });
-                res.send(image);
+                return res.send(image);
+            }
+            catch (err) {
+                console.error("Error saving image record:", err);
+                return res.status(500).json({ error: "Failed to save image record" });
             }
         });
     }
-    catch (error) {
-        sendError(res, req.file.originalname, "create", res.user, error);
+    catch (err) {
+        console.error("Unexpected error:", err);
+        // If req.file exists, use its data; otherwise fallback
+        const fileName = req.file ? req.file.originalname : "unknown filename";
+        return res.status(500).json({
+            error: `Unhandled error while processing '${fileName}'`,
+        });
     }
 });
 // Post User Images

package/models/image.cjs

@@ -291,7 +291,7 @@ module.exports = (sequelize, DataTypes) => {
             if (removed) {
                 image.deleted = true;
                 await image.save();
-                import("../active-citizen/llms/collectionImageGenerator.js").then(async ({ CollectionImageGenerator }) => {
+                import("../active-citizen/llms/imageGeneration/collectionImageGenerator.js").then(async ({ CollectionImageGenerator }) => {
                     try {
                         const mediaManager = new CollectionImageGenerator();
                         await mediaManager.deleteMediaFormatsUrls(image.formats ? JSON.parse(image.formats) : []);

package/models/index.cjs

@@ -48,18 +48,26 @@ if (process.env.NODE_ENV === "production") {
     }
 }
 else {
-    const configPath = path.join(process.cwd(), "ts-out", "config", "config.json");
-    const config = require(configPath)[env];
-    sequelize = new Sequelize(config.database, config.username, config.password, _.merge(config, {
-        dialect: "postgres",
-        minifyAliases: true,
-        dialectOptions: {
-            ssl: false,
-            rejectUnauthorized: false,
-        },
-        logging: false,
-        operatorsAliases: operatorsAliases,
-    }));
+    let config;
+    try {
+        sequelize = new Sequelize(process.env.YP_DEV_DATABASE_NAME, process.env.YP_DEV_DATABASE_USERNAME, process.env.YP_DEV_DATABASE_PASSWORD, {
+            dialect: "postgres",
+            protocol: "postgres",
+            host: process.env.YP_DEV_DATABASE_HOST,
+            port: process.env.YP_DEV_DATABASE_PORT,
+            minifyAliases: true,
+            dialectOptions: {
+                ssl: false,
+                rejectUnauthorized: false,
+            },
+            logging: false,
+            operatorsAliases: operatorsAliases,
+        });
+    }
+    catch (error) {
+        console.error("Error reading or parsing config file:", error);
+        process.exit(1);
+    }
 }
 const db = {};
 async function createCompoundIndexes(indexCommands) {

package/models/video.cjs

@@ -821,7 +821,7 @@ module.exports = (sequelize, DataTypes) => {
                 if (removed) {
                     video.deleted = true;
                     await video.save();
-                    import("../active-citizen/llms/collectionImageGenerator.js").then(async ({ CollectionImageGenerator }) => {
+                    import("../active-citizen/llms/imageGeneration/collectionImageGenerator.js").then(async ({ CollectionImageGenerator }) => {
                         try {
                             const mediaManager = new CollectionImageGenerator();
                             let formats = video.formats;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yrpri/api",
-  "version": "9.0.90",
+  "version": "9.0.92",
   "license": "MIT",
   "author": "Robert Bjarnason & Citizens Foundation",
   "repository": {
@@ -18,50 +18,51 @@
   "dependencies": {
     "@airbrake/node": "^2.1.4",
     "@aws-sdk/client-s3": "^3.617.0",
-    "@azure/openai": "^1.0.0-beta.11",
+    "@google-cloud/aiplatform": "^3.34.0",
     "@google-cloud/speech": "^6.7.0",
-    "@google-cloud/storage": "^7.12.0",
-    "@google-cloud/translate": "^8.0.3",
+    "@google-cloud/storage": "^7.15.0",
+    "@google-cloud/translate": "^8.5.0",
+    "@google-cloud/vertexai": "^1.9.3",
     "@google-cloud/vision": "^4.0.3",
-    "@policysynth/agents": "^1.3.38",
-    "async": "^3.2.0",
+    "@policysynth/agents": "^1.3.64",
+    "async": "^3.2.6",
     "authorized": "^1.0.0",
-    "aws-sdk": "^2.1551.0",
-    "axios": "^1.4.0",
+    "aws-sdk": "^2.1692.0",
+    "axios": "^1.7.9",
     "bcrypt": "^5.1.0",
-    "body-parser": "^1.20.1",
-    "bull": "^4.15.1",
+    "body-parser": "^1.20.3",
+    "bull": "^4.16.5",
     "bunyan": "^1.8.12",
     "bunyan-prettystream": "git+https://github.com/rbjarnason/node-bunyan-prettystream.git",
-    "cheerio": "^1.0.0-rc.12",
+    "cheerio": "^1.0.0",
     "color-hash": "^2.0.1",
     "colors": "^1.4.0",
-    "compression": "^1.7.4",
+    "compression": "^1.8.0",
     "concat-stream": "^2.0.0",
     "connect-redis": "^7.1.0",
-    "cookie-parser": "^1.4.5",
+    "cookie-parser": "^1.4.7",
     "cors": "^2.8.5",
     "deep-equal": "^2.1.0",
     "docx": "^8.5.0",
     "download-file": "^0.1.5",
     "ejs": "^3.1.8",
     "exceljs": "^4.2.0",
-    "express": "^4.18.2",
+    "express": "^4.21.2",
     "express-rate-limit": "^7.4.0",
     "express-session": "git+https://github.com/rbjarnason/session.git#upgrade-21",
     "express-useragent": "^1.0.13",
     "farmhash": "3.3.1",
     "html-to-docx": "^1.8.0",
-    "i18next": "^23.12.2",
+    "i18next": "^24.2.2",
     "i18next-node-fs-backend": "^2.1.3",
     "image-size": "^1.0.2",
     "ip": "^2.0.1",
-    "isbot": "^5.1.13",
+    "isbot": "^5.1.22",
     "iso-639-1": "^3.1.0",
-    "jsonrepair": "^3.5.0",
+    "jsonrepair": "^3.12.0",
     "knuth-shuffle-seeded": "^1.0.6",
     "lodash": "^4.17.15",
-    "marked": "^15.0.4",
+    "marked": "^15.0.7",
     "moment": "^2.24.0",
     "morgan": "~1.10.0",
     "multer": "1.4.5-lts.1",
@@ -71,7 +72,7 @@
     "nodemailer": "^6.9.9",
     "nodemailer-sendgrid": "^1.0.3",
     "open-graph-scraper": "^6.7.0",
-    "openai": "^4.69.0",
+    "openai": "^4.85.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
     "passport-github": "^1.0.0",
@@ -80,24 +81,24 @@
     "passport-openidconnect": "git+https://github.com/rbjarnason/passport-openidconnect.git",
     "passport-saml": "git+https://github.com/rbjarnason/passport-saml.git#smarternj",
     "passport-sso": "git+https://github.com/rbjarnason/passport-sso.git",
-    "pg": "^8.11.3",
+    "pg": "^8.13.3",
     "pg-hstore": "^2.3.4",
     "pgvector": "^0.2.0",
     "ps": "^1.0.0",
     "pug": "^3.0.2",
     "randomstring": "^1.2.3",
     "rate-limit-redis": "^4.2.0",
-    "redis": "^4.6.13",
+    "redis": "^4.7.0",
     "replicate": "^0.32.0",
     "request": "^2.88.2",
     "request-ip": "^3.3.0",
     "sanitize-filename": "^1.6.3",
     "sequelize": "^6.36.0",
     "sequelize-cli": "^6.2.0",
-    "sharp": "^0.33.2",
+    "sharp": "^0.33.5",
     "sitemap": "^7.1.2",
-    "socket.io": "^4.0.0",
-    "stripe": "^17.3.0",
+    "socket.io": "^4.8.1",
+    "stripe": "^17.6.0",
     "striptags": "^3.2.0",
     "uuid": "^10.0.0",
     "weaviate-ts-client": "^2.0.0",
@@ -116,15 +117,15 @@
     "@types/html-docx-js": "^0.3.4",
     "@types/marked": "^6.0.0",
     "@types/morgan": "^1.9.9",
-    "@types/node": "^20.14.12",
+    "@types/node": "^22.13.4",
     "@types/passport": "^1.0.16",
-    "@types/pg": "^8.10.2",
+    "@types/pg": "^8.11.11",
     "@types/request-ip": "^0.0.41",
     "@types/sequelize": "^4.28.9",
     "@types/sharp": "^0.32.0",
     "@types/uuid": "^10.0.0",
     "@types/validator": "^13.11.9",
-    "@types/ws": "^8.5.11",
+    "@types/ws": "^8.5.14",
     "axios": "^1.4.0",
     "colors": "^1.4.0",
     "csv-parse": "^5.3.1",
@@ -163,6 +164,6 @@
     "node_modules"
   ],
   "engines": {
-    "node": "20.11.0"
+    "node": "22.14.0"
   }
 }

package/utils/manifest_generator.cjs

@@ -52,7 +52,6 @@ var generateManifest = function (req, res) {
         "orientation": "any"
     };
     var shortName, name;
-    var domainId = req.ypDomain.id;
     async.series([
         function (seriesCallback) {
             if (req.ypCommunity && req.ypCommunity.configuration &&

package/utils/sitemap_generator.cjs

@@ -12,12 +12,18 @@ const wildCardDomainNames = [
     "betrireykjavik.is",
     "betraisland.is",
     "yrpri.org",
+    "evoly.ai",
     "tarsalgo.net",
     "ypus.org",
     "idea-synergy.com",
     "localhost:4242",
 ];
 var generateSitemap = async function (req, res) {
+    if (!req.ypDomain) {
+        log.error("No domain found in sitemap generation");
+        res.status(500);
+        return;
+    }
     const domainId = req.ypDomain.id;
     const domainName = req.ypDomain.domain_name;
     let community = req.ypCommunity && req.ypCommunity.id ? req.ypCommunity : null;