@yrpri/api 9.0.220 → 9.0.221

package/services/engine/allOurIdeas/aiHelper.d.ts

@@ -1,5 +1,4 @@
 import { OpenAI } from "openai";
-import { Stream } from "openai/streaming";
 import { WebSocket } from "ws";
 export declare class AiHelper {
     openaiClient: OpenAI;
@@ -16,7 +15,7 @@ export declare class AiHelper {
     getModerationResponse: (instructions: string, question: string, answerToModerate: string) => Promise<boolean>;
     streamChatCompletions(messages: any[]): Promise<void>;
     sendToClient(sender: string, message: string, type?: string): void;
-    streamWebSocketResponses(stream: Stream<OpenAI.Chat.Completions.ChatCompletionChunk>): Promise<void>;
+    streamWebSocketResponses(stream: AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>): Promise<void>;
     getAnswerIdeas(question: string, previousIdeas: string[] | null, firstMessage: string | null): Promise<string | null | undefined>;
     getAiAnalysis(questionId: number, contextPrompt: string, answers: AoiChoiceData[], cacheKeyForFullResponse: string, redisClient: any, locale: string, topOrBottomIdeasText: string, typeOfAnalysisText: string): Promise<string | null | undefined>;
 }

package/services/engine/analytics/manager.cjs

@@ -6,7 +6,7 @@ const importCommunity = require('./utils.cjs').importCommunity;
 const importGroup = require('./utils.cjs').importGroup;
 const importPost = require('./utils.cjs').importPost;
 const importPoint = require('./utils.cjs').importPoint;
-const request = require('request');
+const request = require("../../../utils/requestCompat.cjs");
 const updateDomain = (domainId, done) => {
     log.info('updateDomain');
     models.Domain.unscoped().findOne({

package/services/engine/analytics/plausible/manager.cjs

@@ -1,7 +1,7 @@
 "use strict";
 const models = require("../../../../models/index.cjs");
 const log = require("../../../../utils/logger.cjs");
-const request = require("request");
+const request = require("../../../../utils/requestCompat.cjs");
 const moment = require("moment");
 // This SQL is needed to allow the site API
 // UPDATE api_keys SET scopes = '{sites:provision:*}' WHERE name = 'Development';

package/services/engine/analytics/utils.cjs

@@ -1,6 +1,6 @@
 "use strict";
 const _ = require('lodash');
-const request = require('request');
+const request = require("../../../utils/requestCompat.cjs");
 const models = require('../../../models/index.cjs');
 const log = require('../../../utils/logger.cjs');
 const convertToString = (integer, type) => {

package/services/engine/notifications/emails_utils.cjs

@@ -242,7 +242,9 @@ const filterNotificationForDelivery = function (notification, user, template, su
                 }
                 else {
                     const redisKey = `${SUPPRESSION_KEYBASE}${user.id}`;
-                    redisConnection.get(redisKey, (error, found) => {
+                    redisConnection
+                        .get(redisKey)
+                        .then((found) => {
                         if (found) {
                             log.info(`Suppressing emails for user ${user.email} settings ${LIMIT_EMAILS_FOR_SECONDS}`);
                             callback();
@@ -250,6 +252,13 @@ const filterNotificationForDelivery = function (notification, user, template, su
                         else {
                             processNotification(notification, user, template, subject, callback);
                         }
+                    })
+                        .catch((error) => {
+                        log.warn("Redis suppression lookup failed, continuing delivery", {
+                            err: error,
+                            userId: user.id,
+                        });
+                        processNotification(notification, user, template, subject, callback);
                     });
                 }
             });

package/services/engine/recommendations/events_manager.cjs

@@ -3,7 +3,7 @@ const models = require('../../../models/index.cjs');
 const _ = require('lodash');
 const async = require('async');
 const log = require('../../../utils/logger.cjs');
-const request = require('request');
+const request = require("../../../utils/requestCompat.cjs");
 let airbrake = null;
 if (process.env.AIRBRAKE_PROJECT_ID) {
     airbrake = require('../../utils/airbrake.cjs');

package/services/engine/reports/common_utils.cjs

@@ -6,7 +6,7 @@ const moment = require('moment');
 const skipEmail = false;
 const aws = require('aws-sdk');
 const log = require('../../../utils/logger.cjs');
-const request = require('request');
+const request = require("../../../utils/requestCompat.cjs");
 const fs = require('fs');
 const downloadImage = (uri, filename, callback) => {
     request.head(uri, (err, res, body) => {

package/services/llms/baseChatBot.d.ts

@@ -1,5 +1,4 @@
 import { OpenAI } from "openai";
-import { Stream } from "openai/streaming.js";
 import WebSocket from "ws";
 import ioredis from "ioredis";
 export declare class YpBaseChatBot {
@@ -26,7 +25,7 @@ export declare class YpBaseChatBot {
     sendAgentCompleted(name: string, lastAgent?: boolean, error?: string | undefined): void;
     sendAgentUpdate(message: string): void;
     sendToClient(sender: YpSenderType, message: string, type?: YpAssistantMessageType, uniqueToken?: string | undefined, hiddenContextMessage?: boolean): void;
-    streamWebSocketResponses(stream: Stream<OpenAI.Chat.Completions.ChatCompletionChunk>): Promise<void>;
+    streamWebSocketResponses(stream: AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>): Promise<void>;
     saveMemoryIfNeeded(): Promise<void>;
     setChatLog(chatLog: YpSimpleChatLog[]): Promise<void>;
     conversation(chatLog: YpSimpleChatLog[]): Promise<void>;

package/services/llms/imageGeneration/s3Service.js

@@ -37,13 +37,17 @@ export class S3Service {
         const params = {
             Bucket: bucket,
             Key: key,
-            ACL: "private",
         };
         log.info(`Disabling/Deleting Key from S3: ${JSON.stringify(params)}`);
         return new Promise((resolve, reject) => {
-            s3.putObjectAcl(params, (err, data) => {
+            s3.deleteObject(params, (err, data) => {
                 if (err) {
-                    log.error(`Error deleting image from S3: ${err}`);
+                    log.error("Error deleting image from S3", {
+                        imageUrl,
+                        bucket,
+                        key,
+                        err,
+                    });
                     reject(err);
                 }
                 else {
@@ -87,25 +91,35 @@ export class S3Service {
         let bucket, key;
         const cfImageProxyDomain = process.env.CLOUDFLARE_IMAGE_PROXY_DOMAIN;
         const s3Bucket = process.env.S3_BUCKET;
+        const parsedUrl = new URL(imageUrl);
         if (cfImageProxyDomain && imageUrl.includes(cfImageProxyDomain)) {
-            const urlPath = new URL(imageUrl).pathname;
-            const [, ...pathParts] = urlPath.split("/");
+            const [, ...pathParts] = parsedUrl.pathname.split("/");
             bucket = s3Bucket;
             key = pathParts.join("/");
         }
+        else if (parsedUrl.hostname === process.env.S3_ENDPOINT) {
+            const [, maybeBucket, ...pathParts] = parsedUrl.pathname.split("/");
+            bucket = process.env.MINIO_ROOT_USER ? maybeBucket || s3Bucket : s3Bucket;
+            key = process.env.MINIO_ROOT_USER ? pathParts.join("/") : parsedUrl.pathname.slice(1);
+        }
         else {
-            const match = imageUrl.match(/https:\/\/(.+?)\.s3\.amazonaws\.com\/(.+)/);
-            if (match) {
-                bucket = match[1];
-                key = match[2];
+            const hostParts = parsedUrl.hostname.split(".");
+            if (hostParts.length >= 4 && hostParts[1] === "s3") {
+                bucket = hostParts[0];
+                key = parsedUrl.pathname.replace(/^\/+/, "");
             }
         }
         return { bucket, key };
     }
     async deleteMediaFormatsUrls(formats) {
         for (const url of formats) {
-            await this.deleteS3Url(url);
-            log.info(`Deleted image from S3: ${url}`);
+            try {
+                await this.deleteS3Url(url);
+                log.info(`Deleted image from S3: ${url}`);
+            }
+            catch (error) {
+                log.warn("Best-effort image cleanup failed", { url, error });
+            }
         }
     }
 }

package/services/scripts/translation_replace_text_from_url.js

@@ -2,7 +2,7 @@ const models = require('../../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../../utils/requestCompat.cjs");
 const farmhash = require('farmhash');
 //const communityId = "1264"; //process.argv[2];
 //const communityId = "1402"; //process.argv[2];

package/services/utils/redisConnection.cjs

@@ -9,18 +9,17 @@ if (process.env.REDIS_URL) {
     }
     if (redisUrl.includes("rediss://")) {
         redisClient = redis.createClient({
-            legacyMode: false,
             url: redisUrl,
             pingInterval: 10000,
             socket: { tls: true, rejectUnauthorized: false },
         });
     }
     else {
-        redisClient = redis.createClient({ legacyMode: true, url: redisUrl });
+        redisClient = redis.createClient({ url: redisUrl });
     }
 }
 else {
-    redisClient = redis.createClient({ legacyMode: true });
+    redisClient = redis.createClient();
 }
 redisClient.on("error", (err) => log.error("Backend Redis client error", err));
 redisClient.on("connect", () => log.info("Backend Redis client is connect"));

package/services/utils/translation_cloning.cjs

@@ -3,7 +3,7 @@ const models = require('../../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../../utils/requestCompat.cjs");
 const farmhash = require('farmhash');
 const fixTargetLocale = require('./translation_helpers.cjs').fixTargetLocale;
 // For post get translations in all locales (that exists)

package/services/utils/translation_helpers.cjs

@@ -3,7 +3,7 @@ const models = require("../../models/index.cjs");
 const async = require("async");
 const _ = require("lodash");
 const fs = require("fs");
-const request = require("request");
+const request = require("../../utils/requestCompat.cjs");
 const farmhash = require("farmhash");
 const log = require('../../utils/logger.cjs');
 const fixTargetLocale = (itemTargetLocale) => {

package/tests/emails_utils.test.cjs

@@ -0,0 +1,130 @@
+"use strict";
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const vm = require("node:vm");
+const Module = require("node:module");
+const emailsUtilsPath = path.resolve(__dirname, "../services/engine/notifications/emails_utils.cjs");
+const loadEmailsUtils = ({ logger, redisConnection, queue, models }) => {
+    const source = fs.readFileSync(emailsUtilsPath, "utf8");
+    const module = { exports: {} };
+    const baseRequire = Module.createRequire(emailsUtilsPath);
+    const mockedRequire = (request) => {
+        switch (request) {
+            case "../../../utils/logger.cjs":
+                return logger;
+            case "../../utils/redisConnection.cjs":
+                return redisConnection;
+            case "../../workers/queue.cjs":
+                return queue;
+            case "../../../models/index.cjs":
+                return models;
+            case "nodemailer":
+                return {
+                    createTransport() {
+                        return {
+                            verify(callback) {
+                                if (callback) {
+                                    callback(null, true);
+                                }
+                            },
+                        };
+                    },
+                };
+            case "../../utils/i18n.cjs":
+                return {
+                    t(value) {
+                        return value;
+                    },
+                };
+            default:
+                return baseRequire(request);
+        }
+    };
+    const wrapped = Module.wrap(source);
+    const compiled = vm.runInThisContext(wrapped, {
+        filename: emailsUtilsPath,
+    });
+    compiled(module.exports, mockedRequire, module, emailsUtilsPath, path.dirname(emailsUtilsPath));
+    return module.exports;
+};
+test("filterNotificationForDelivery falls back to delivery when Redis suppression lookup fails", async () => {
+    const queuedEmails = [];
+    const redisSetExCalls = [];
+    const warnings = [];
+    const emailsUtils = loadEmailsUtils({
+        logger: {
+            info() { },
+            error() { },
+            warn(message, metadata) {
+                warnings.push({ message, metadata });
+            },
+        },
+        redisConnection: {
+            async get() {
+                throw new Error("redis down");
+            },
+            setEx(...args) {
+                redisSetExCalls.push(args);
+            },
+        },
+        queue: {
+            add(...args) {
+                queuedEmails.push(args);
+            },
+        },
+        models: {
+            AcNotification: {
+                METHOD_MUTED: "muted",
+                FREQUENCY_AS_IT_HAPPENS: "as_it_happens",
+            },
+        },
+    });
+    const group = {
+        name: "visible-group",
+        async hasGroupAdmins() {
+            return false;
+        },
+    };
+    const user = {
+        id: 42,
+        email: "person@example.com",
+        notifications_settings: {
+            point_activity: {
+                method: "email",
+                frequency: "as_it_happens",
+            },
+        },
+    };
+    const notification = {
+        from_notification_setting: "point_activity",
+        AcActivities: [
+            {
+                Domain: { id: 1 },
+                Community: null,
+                Group: group,
+                Point: {
+                    Group: group,
+                },
+                Post: null,
+            },
+        ],
+    };
+    await new Promise((resolve, reject) => {
+        emailsUtils.filterNotificationForDelivery(notification, user, "point_activity", "A subject", (error) => {
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+    assert.equal(warnings.length, 1);
+    assert.equal(warnings[0].message, "Redis suppression lookup failed, continuing delivery");
+    assert.equal(warnings[0].metadata.userId, 42);
+    assert.equal(queuedEmails.length, 1);
+    assert.equal(queuedEmails[0][0], "send-one-email");
+    assert.equal(redisSetExCalls.length, 1);
+});

package/tests/emails_utils.test.d.cts

@@ -0,0 +1 @@
+export {};

package/tests/multerSharpS3Compat.test.cjs

@@ -0,0 +1,229 @@
+"use strict";
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const Module = require("node:module");
+const { PassThrough } = require("node:stream");
+const sharpPath = require.resolve("sharp");
+const mimeTypesPath = require.resolve("mime-types");
+const multerSharpS3CompatPath = require.resolve("../utils/multerSharpS3Compat.cjs");
+const injectMockModule = (modulePath, moduleExports) => {
+    const mockModule = new Module(modulePath);
+    mockModule.filename = modulePath;
+    mockModule.loaded = true;
+    mockModule.exports = moduleExports;
+    require.cache[modulePath] = mockModule;
+};
+const loadMulterSharpS3Compat = ({ mockSharp, mockMimeTypes }) => {
+    const originalSharpModule = require.cache[sharpPath];
+    const originalMimeTypesModule = require.cache[mimeTypesPath];
+    const originalCompatModule = require.cache[multerSharpS3CompatPath];
+    injectMockModule(sharpPath, mockSharp);
+    injectMockModule(mimeTypesPath, mockMimeTypes);
+    delete require.cache[multerSharpS3CompatPath];
+    const multerSharpS3Compat = require(multerSharpS3CompatPath);
+    return {
+        multerSharpS3Compat,
+        restore() {
+            delete require.cache[multerSharpS3CompatPath];
+            if (originalSharpModule) {
+                require.cache[sharpPath] = originalSharpModule;
+            }
+            else {
+                delete require.cache[sharpPath];
+            }
+            if (originalMimeTypesModule) {
+                require.cache[mimeTypesPath] = originalMimeTypesModule;
+            }
+            else {
+                delete require.cache[mimeTypesPath];
+            }
+            if (originalCompatModule) {
+                require.cache[multerSharpS3CompatPath] = originalCompatModule;
+            }
+        },
+    };
+};
+const createMockSharp = (requestedFormats) => {
+    return () => {
+        let selectedFormat = null;
+        const stream = new PassThrough();
+        stream.resize = () => stream;
+        stream.toFormat = (format) => {
+            selectedFormat = format;
+            requestedFormats.push(format);
+            return stream;
+        };
+        process.nextTick(() => {
+            stream.end(Buffer.from(`formatted:${selectedFormat || "original"}`));
+        });
+        return stream;
+    };
+};
+const createMockS3 = ({ failKeys = [] } = {}) => {
+    const uploads = [];
+    const deletedObjects = [];
+    return {
+        uploads,
+        deletedObjects,
+        s3: {
+            upload(params) {
+                uploads.push(params);
+                return {
+                    promise() {
+                        return new Promise((resolve, reject) => {
+                            const chunks = [];
+                            params.Body.on("data", (chunk) => {
+                                chunks.push(chunk);
+                            });
+                            params.Body.on("error", reject);
+                            params.Body.on("end", () => {
+                                if (failKeys.includes(params.Key)) {
+                                    reject(new Error(`upload failed for ${params.Key}`));
+                                }
+                                else {
+                                    resolve({
+                                        Bucket: params.Bucket,
+                                        Key: params.Key,
+                                        Location: `https://example.com/${params.Key}`,
+                                        uploadedBody: Buffer.concat(chunks).toString("utf8"),
+                                    });
+                                }
+                            });
+                        });
+                    },
+                };
+            },
+            deleteObject(params) {
+                deletedObjects.push(params);
+                return {
+                    promise() {
+                        return Promise.resolve();
+                    },
+                };
+            },
+        },
+    };
+};
+test("multerSharpS3Compat honors file.outputFormat over the default format", async (t) => {
+    const requestedFormats = [];
+    const { s3, uploads } = createMockS3();
+    const { multerSharpS3Compat, restore } = loadMulterSharpS3Compat({
+        mockSharp: createMockSharp(requestedFormats),
+        mockMimeTypes: {
+            contentType(format) {
+                return `image/${format}`;
+            },
+        },
+    });
+    t.after(restore);
+    const storage = multerSharpS3Compat({
+        s3,
+        Bucket: "images",
+        toFormat: "png",
+        Key: "animated.gif",
+    });
+    const fileStream = new PassThrough();
+    fileStream.end(Buffer.from("source"));
+    const file = {
+        mimetype: "image/gif",
+        originalname: "animated.gif",
+        outputFormat: "gif",
+        stream: fileStream,
+    };
+    const result = await new Promise((resolve, reject) => {
+        storage._handleFile({}, file, (error, value) => {
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve(value);
+            }
+        });
+    });
+    assert.deepEqual(requestedFormats, ["gif"]);
+    assert.equal(uploads.length, 1);
+    assert.equal(uploads[0].ContentType, "image/gif");
+    assert.equal(result.default.ContentType, "image/gif");
+    assert.equal(result.mimetype, "image/gif");
+});
+test("multerSharpS3Compat rolls back earlier variants when a later upload fails", async (t) => {
+    const requestedFormats = [];
+    const failingKey = "up/animated-large-2.gif";
+    const { s3, deletedObjects } = createMockS3({ failKeys: [failingKey] });
+    const { multerSharpS3Compat, restore } = loadMulterSharpS3Compat({
+        mockSharp: createMockSharp(requestedFormats),
+        mockMimeTypes: {
+            contentType(format) {
+                return `image/${format}`;
+            },
+        },
+    });
+    t.after(restore);
+    const storage = multerSharpS3Compat({
+        s3,
+        Bucket: "images",
+        toFormat: "png",
+        Key: "animated.gif",
+        multiple: true,
+        resize: [
+            { suffix: "-small-1", directory: "up" },
+            { suffix: "-large-2", directory: "up" },
+        ],
+    });
+    const fileStream = new PassThrough();
+    fileStream.end(Buffer.from("source"));
+    const file = {
+        mimetype: "image/gif",
+        originalname: "animated.gif",
+        outputFormat: "gif",
+        stream: fileStream,
+    };
+    const error = await new Promise((resolve) => {
+        storage._handleFile({}, file, (uploadError) => {
+            resolve(uploadError);
+        });
+    });
+    assert.match(error.message, /upload failed/);
+    assert.deepEqual(requestedFormats, ["gif", "gif"]);
+    assert.deepEqual(deletedObjects, [
+        {
+            Bucket: "images",
+            Key: "up/animated-small-1.gif",
+        },
+    ]);
+});
+test("_removeFile deletes every uploaded variant in a multi-variant upload", async (t) => {
+    const { s3, deletedObjects } = createMockS3();
+    const { multerSharpS3Compat, restore } = loadMulterSharpS3Compat({
+        mockSharp: createMockSharp([]),
+        mockMimeTypes: {
+            contentType(format) {
+                return `image/${format}`;
+            },
+        },
+    });
+    t.after(restore);
+    const storage = multerSharpS3Compat({
+        s3,
+        Bucket: "images",
+        Key: "animated.gif",
+    });
+    await new Promise((resolve, reject) => {
+        storage._removeFile({}, {
+            default: { Bucket: "images", Key: "up/a.gif" },
+            "-small-1": { Bucket: "images", Key: "up/a-small.gif" },
+            mimetype: "image/gif",
+        }, (error) => {
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+    assert.deepEqual(deletedObjects, [
+        { Bucket: "images", Key: "up/a.gif" },
+        { Bucket: "images", Key: "up/a-small.gif" },
+    ]);
+});

package/tests/multerSharpS3Compat.test.d.cts

@@ -0,0 +1 @@
+export {};