@yrpri/api 9.0.220 → 9.0.222

package/scripts/endorsementFraudDetection/bulkDeleteDuplicateEndorsmentsFromUrl.js

@@ -2,7 +2,7 @@ const models = require('../../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../../utils/requestCompat.cjs");
 const recountCommunity = require('../../utils/recount_utils.cjs').recountCommunity;
 const recountPost = require('../../utils/recount_utils.cjs').recountPost;
 const communityId = process.argv[2];

package/scripts/landUseGame/export3Ddata.js

@@ -1,4 +1,4 @@
-const { post } = require("request");
+const { post } = require("../../utils/requestCompat.cjs");
 const queue = require("../../services/workers/queue.cjs");
 const models = require("../../models/index.cjs");
 const fs = require("fs");

package/scripts/movePostsToGroupsRecountGroupFromUrl.js

@@ -2,7 +2,7 @@ const models = require('../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../utils/requestCompat.cjs");
 /*
 const urlToConfig = "https://yrpri-eu-direct-assets.s3-eu-west-1.amazonaws.com/WBMoveIdeas160221.csv"//process.argv[1];
 const urlToAddAddFront = "https://kyrgyz-aris.yrpri.org/"; // process.argv[2];

package/scripts/recountALLCommunityGroupCounts.js

@@ -2,7 +2,7 @@ const models = require('../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../utils/requestCompat.cjs");
 models.Community.findAll({
     attributes: ['id', 'counter_groups']
 }).then(allCommunities => {

package/scripts/recountCommunitesFromUrl.js

@@ -2,7 +2,7 @@ const models = require('../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../utils/requestCompat.cjs");
 const recountCommunity = require('../utils/recount_utils.cjs').recountCommunity;
 /*
 const urlToConfig = "https://yrpri-eu-direct-assets.s3-eu-west-1.amazonaws.com/WBMoveIdeas160221.csv"//process.argv[1];

package/scripts/recountCommunity.js

@@ -2,7 +2,7 @@ const models = require('../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../utils/requestCompat.cjs");
 const recountCommunity = require('../utils/recount_utils.cjs').recountCommunity;
 /*
 const urlToConfig = "https://yrpri-eu-direct-assets.s3-eu-west-1.amazonaws.com/WBMoveIdeas160221.csv"//process.argv[1];

package/scripts/setLanguageOnGroupCommunitesFromUrl.js

@@ -2,7 +2,7 @@ const models = require('../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../utils/requestCompat.cjs");
 /*
 const urlToConfig = "https://yrpri-eu-direct-assets.s3-eu-west-1.amazonaws.com/WB_ResetLanguageOnCommuntiesList_16_02_21.csv"//process.argv[1];
 const toLanguage = "ru"; // process.argv[2];

package/services/engine/allOurIdeas/aiHelper.d.ts

@@ -1,5 +1,4 @@
 import { OpenAI } from "openai";
-import { Stream } from "openai/streaming";
 import { WebSocket } from "ws";
 export declare class AiHelper {
     openaiClient: OpenAI;
@@ -16,7 +15,7 @@ export declare class AiHelper {
     getModerationResponse: (instructions: string, question: string, answerToModerate: string) => Promise<boolean>;
     streamChatCompletions(messages: any[]): Promise<void>;
     sendToClient(sender: string, message: string, type?: string): void;
-    streamWebSocketResponses(stream: Stream<OpenAI.Chat.Completions.ChatCompletionChunk>): Promise<void>;
+    streamWebSocketResponses(stream: AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>): Promise<void>;
     getAnswerIdeas(question: string, previousIdeas: string[] | null, firstMessage: string | null): Promise<string | null | undefined>;
     getAiAnalysis(questionId: number, contextPrompt: string, answers: AoiChoiceData[], cacheKeyForFullResponse: string, redisClient: any, locale: string, topOrBottomIdeasText: string, typeOfAnalysisText: string): Promise<string | null | undefined>;
 }

package/services/engine/analytics/manager.cjs

@@ -6,7 +6,7 @@ const importCommunity = require('./utils.cjs').importCommunity;
 const importGroup = require('./utils.cjs').importGroup;
 const importPost = require('./utils.cjs').importPost;
 const importPoint = require('./utils.cjs').importPoint;
-const request = require('request');
+const request = require("../../../utils/requestCompat.cjs");
 const updateDomain = (domainId, done) => {
     log.info('updateDomain');
     models.Domain.unscoped().findOne({

package/services/engine/analytics/plausible/manager.cjs

@@ -1,7 +1,7 @@
 "use strict";
 const models = require("../../../../models/index.cjs");
 const log = require("../../../../utils/logger.cjs");
-const request = require("request");
+const request = require("../../../../utils/requestCompat.cjs");
 const moment = require("moment");
 // This SQL is needed to allow the site API
 // UPDATE api_keys SET scopes = '{sites:provision:*}' WHERE name = 'Development';

package/services/engine/analytics/utils.cjs

@@ -1,6 +1,6 @@
 "use strict";
 const _ = require('lodash');
-const request = require('request');
+const request = require("../../../utils/requestCompat.cjs");
 const models = require('../../../models/index.cjs');
 const log = require('../../../utils/logger.cjs');
 const convertToString = (integer, type) => {

package/services/engine/notifications/emails_utils.cjs

@@ -242,7 +242,9 @@ const filterNotificationForDelivery = function (notification, user, template, su
                 }
                 else {
                     const redisKey = `${SUPPRESSION_KEYBASE}${user.id}`;
-                    redisConnection.get(redisKey, (error, found) => {
+                    redisConnection
+                        .get(redisKey)
+                        .then((found) => {
                         if (found) {
                             log.info(`Suppressing emails for user ${user.email} settings ${LIMIT_EMAILS_FOR_SECONDS}`);
                             callback();
@@ -250,6 +252,13 @@ const filterNotificationForDelivery = function (notification, user, template, su
                         else {
                             processNotification(notification, user, template, subject, callback);
                         }
+                    })
+                        .catch((error) => {
+                        log.warn("Redis suppression lookup failed, continuing delivery", {
+                            err: error,
+                            userId: user.id,
+                        });
+                        processNotification(notification, user, template, subject, callback);
                     });
                 }
             });

package/services/engine/recommendations/events_manager.cjs

@@ -3,7 +3,7 @@ const models = require('../../../models/index.cjs');
 const _ = require('lodash');
 const async = require('async');
 const log = require('../../../utils/logger.cjs');
-const request = require('request');
+const request = require("../../../utils/requestCompat.cjs");
 let airbrake = null;
 if (process.env.AIRBRAKE_PROJECT_ID) {
     airbrake = require('../../utils/airbrake.cjs');

package/services/engine/reports/common_utils.cjs

@@ -6,7 +6,7 @@ const moment = require('moment');
 const skipEmail = false;
 const aws = require('aws-sdk');
 const log = require('../../../utils/logger.cjs');
-const request = require('request');
+const request = require("../../../utils/requestCompat.cjs");
 const fs = require('fs');
 const downloadImage = (uri, filename, callback) => {
     request.head(uri, (err, res, body) => {

package/services/llms/baseChatBot.d.ts

@@ -1,5 +1,4 @@
 import { OpenAI } from "openai";
-import { Stream } from "openai/streaming.js";
 import WebSocket from "ws";
 import ioredis from "ioredis";
 export declare class YpBaseChatBot {
@@ -26,7 +25,7 @@ export declare class YpBaseChatBot {
     sendAgentCompleted(name: string, lastAgent?: boolean, error?: string | undefined): void;
     sendAgentUpdate(message: string): void;
     sendToClient(sender: YpSenderType, message: string, type?: YpAssistantMessageType, uniqueToken?: string | undefined, hiddenContextMessage?: boolean): void;
-    streamWebSocketResponses(stream: Stream<OpenAI.Chat.Completions.ChatCompletionChunk>): Promise<void>;
+    streamWebSocketResponses(stream: AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>): Promise<void>;
     saveMemoryIfNeeded(): Promise<void>;
     setChatLog(chatLog: YpSimpleChatLog[]): Promise<void>;
     conversation(chatLog: YpSimpleChatLog[]): Promise<void>;

package/services/llms/imageGeneration/s3Service.js

@@ -2,6 +2,49 @@ import AWS from "aws-sdk";
 import fs from "fs";
 import axios from "axios";
 import log from "../../../utils/loggerTs.js";
+const normalizeEndpointHost = (endpoint) => {
+    if (!endpoint) {
+        return null;
+    }
+    try {
+        const normalizedUrl = new URL(endpoint.includes("://") ? endpoint : `https://${endpoint}`);
+        return {
+            host: normalizedUrl.host,
+            hostname: normalizedUrl.hostname,
+        };
+    }
+    catch {
+        return {
+            host: endpoint,
+            hostname: endpoint,
+        };
+    }
+};
+const getBucketFromVirtualHost = (parsedUrl, normalizedEndpoint) => {
+    if (!normalizedEndpoint) {
+        return null;
+    }
+    const hostSuffix = `.${normalizedEndpoint.host}`;
+    const hostnameSuffix = `.${normalizedEndpoint.hostname}`;
+    if (parsedUrl.host.endsWith(hostSuffix)) {
+        return parsedUrl.host.slice(0, -hostSuffix.length) || null;
+    }
+    if (parsedUrl.hostname.endsWith(hostnameSuffix)) {
+        return parsedUrl.hostname.slice(0, -hostnameSuffix.length) || null;
+    }
+    return null;
+};
+const getAwsBucketFromHostname = (hostname) => {
+    const s3MarkerIndex = hostname.indexOf(".s3.");
+    if (s3MarkerIndex > 0) {
+        return hostname.slice(0, s3MarkerIndex) || null;
+    }
+    const legacyS3MarkerIndex = hostname.indexOf(".s3-");
+    if (legacyS3MarkerIndex > 0) {
+        return hostname.slice(0, legacyS3MarkerIndex) || null;
+    }
+    return null;
+};
 export class S3Service {
     constructor(cloudflareApiKey, cloudflareZoneId) {
         this.cloudflareApiKey = cloudflareApiKey;
@@ -37,13 +80,17 @@ export class S3Service {
         const params = {
             Bucket: bucket,
             Key: key,
-            ACL: "private",
         };
         log.info(`Disabling/Deleting Key from S3: ${JSON.stringify(params)}`);
         return new Promise((resolve, reject) => {
-            s3.putObjectAcl(params, (err, data) => {
+            s3.deleteObject(params, (err, data) => {
                 if (err) {
-                    log.error(`Error deleting image from S3: ${err}`);
+                    log.error("Error deleting image from S3", {
+                        imageUrl,
+                        bucket,
+                        key,
+                        err,
+                    });
                     reject(err);
                 }
                 else {
@@ -87,25 +134,39 @@ export class S3Service {
         let bucket, key;
         const cfImageProxyDomain = process.env.CLOUDFLARE_IMAGE_PROXY_DOMAIN;
         const s3Bucket = process.env.S3_BUCKET;
+        const normalizedEndpoint = normalizeEndpointHost(process.env.S3_ENDPOINT);
+        const parsedUrl = new URL(imageUrl);
         if (cfImageProxyDomain && imageUrl.includes(cfImageProxyDomain)) {
-            const urlPath = new URL(imageUrl).pathname;
-            const [, ...pathParts] = urlPath.split("/");
+            const [, ...pathParts] = parsedUrl.pathname.split("/");
             bucket = s3Bucket;
             key = pathParts.join("/");
         }
+        else if (normalizedEndpoint &&
+            (parsedUrl.hostname === normalizedEndpoint.hostname ||
+                parsedUrl.host === normalizedEndpoint.host)) {
+            const [, maybeBucket, ...pathParts] = parsedUrl.pathname.split("/");
+            bucket = process.env.MINIO_ROOT_USER ? maybeBucket || s3Bucket : s3Bucket;
+            key = process.env.MINIO_ROOT_USER ? pathParts.join("/") : parsedUrl.pathname.slice(1);
+        }
         else {
-            const match = imageUrl.match(/https:\/\/(.+?)\.s3\.amazonaws\.com\/(.+)/);
-            if (match) {
-                bucket = match[1];
-                key = match[2];
+            const virtualHostBucket = getBucketFromVirtualHost(parsedUrl, normalizedEndpoint) ||
+                getAwsBucketFromHostname(parsedUrl.hostname);
+            if (virtualHostBucket) {
+                bucket = virtualHostBucket;
+                key = parsedUrl.pathname.replace(/^\/+/, "");
             }
         }
         return { bucket, key };
     }
     async deleteMediaFormatsUrls(formats) {
         for (const url of formats) {
-            await this.deleteS3Url(url);
-            log.info(`Deleted image from S3: ${url}`);
+            try {
+                await this.deleteS3Url(url);
+                log.info(`Deleted image from S3: ${url}`);
+            }
+            catch (error) {
+                log.warn("Best-effort image cleanup failed", { url, error });
+            }
         }
     }
 }

package/services/scripts/translation_replace_text_from_url.js

@@ -2,7 +2,7 @@ const models = require('../../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../../utils/requestCompat.cjs");
 const farmhash = require('farmhash');
 //const communityId = "1264"; //process.argv[2];
 //const communityId = "1402"; //process.argv[2];

package/services/utils/redisConnection.cjs

@@ -9,18 +9,17 @@ if (process.env.REDIS_URL) {
     }
     if (redisUrl.includes("rediss://")) {
         redisClient = redis.createClient({
-            legacyMode: false,
             url: redisUrl,
             pingInterval: 10000,
             socket: { tls: true, rejectUnauthorized: false },
         });
     }
     else {
-        redisClient = redis.createClient({ legacyMode: true, url: redisUrl });
+        redisClient = redis.createClient({ url: redisUrl });
     }
 }
 else {
-    redisClient = redis.createClient({ legacyMode: true });
+    redisClient = redis.createClient();
 }
 redisClient.on("error", (err) => log.error("Backend Redis client error", err));
 redisClient.on("connect", () => log.info("Backend Redis client is connect"));

package/services/utils/translation_cloning.cjs

@@ -3,7 +3,7 @@ const models = require('../../models/index.cjs');
 const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
-const request = require('request');
+const request = require("../../utils/requestCompat.cjs");
 const farmhash = require('farmhash');
 const fixTargetLocale = require('./translation_helpers.cjs').fixTargetLocale;
 // For post get translations in all locales (that exists)

package/services/utils/translation_helpers.cjs

@@ -3,7 +3,7 @@ const models = require("../../models/index.cjs");
 const async = require("async");
 const _ = require("lodash");
 const fs = require("fs");
-const request = require("request");
+const request = require("../../utils/requestCompat.cjs");
 const farmhash = require("farmhash");
 const log = require('../../utils/logger.cjs');
 const fixTargetLocale = (itemTargetLocale) => {

package/tests/emails_utils.test.cjs

@@ -0,0 +1,130 @@
+"use strict";
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const vm = require("node:vm");
+const Module = require("node:module");
+const emailsUtilsPath = path.resolve(__dirname, "../services/engine/notifications/emails_utils.cjs");
+const loadEmailsUtils = ({ logger, redisConnection, queue, models }) => {
+    const source = fs.readFileSync(emailsUtilsPath, "utf8");
+    const module = { exports: {} };
+    const baseRequire = Module.createRequire(emailsUtilsPath);
+    const mockedRequire = (request) => {
+        switch (request) {
+            case "../../../utils/logger.cjs":
+                return logger;
+            case "../../utils/redisConnection.cjs":
+                return redisConnection;
+            case "../../workers/queue.cjs":
+                return queue;
+            case "../../../models/index.cjs":
+                return models;
+            case "nodemailer":
+                return {
+                    createTransport() {
+                        return {
+                            verify(callback) {
+                                if (callback) {
+                                    callback(null, true);
+                                }
+                            },
+                        };
+                    },
+                };
+            case "../../utils/i18n.cjs":
+                return {
+                    t(value) {
+                        return value;
+                    },
+                };
+            default:
+                return baseRequire(request);
+        }
+    };
+    const wrapped = Module.wrap(source);
+    const compiled = vm.runInThisContext(wrapped, {
+        filename: emailsUtilsPath,
+    });
+    compiled(module.exports, mockedRequire, module, emailsUtilsPath, path.dirname(emailsUtilsPath));
+    return module.exports;
+};
+test("filterNotificationForDelivery falls back to delivery when Redis suppression lookup fails", async () => {
+    const queuedEmails = [];
+    const redisSetExCalls = [];
+    const warnings = [];
+    const emailsUtils = loadEmailsUtils({
+        logger: {
+            info() { },
+            error() { },
+            warn(message, metadata) {
+                warnings.push({ message, metadata });
+            },
+        },
+        redisConnection: {
+            async get() {
+                throw new Error("redis down");
+            },
+            setEx(...args) {
+                redisSetExCalls.push(args);
+            },
+        },
+        queue: {
+            add(...args) {
+                queuedEmails.push(args);
+            },
+        },
+        models: {
+            AcNotification: {
+                METHOD_MUTED: "muted",
+                FREQUENCY_AS_IT_HAPPENS: "as_it_happens",
+            },
+        },
+    });
+    const group = {
+        name: "visible-group",
+        async hasGroupAdmins() {
+            return false;
+        },
+    };
+    const user = {
+        id: 42,
+        email: "person@example.com",
+        notifications_settings: {
+            point_activity: {
+                method: "email",
+                frequency: "as_it_happens",
+            },
+        },
+    };
+    const notification = {
+        from_notification_setting: "point_activity",
+        AcActivities: [
+            {
+                Domain: { id: 1 },
+                Community: null,
+                Group: group,
+                Point: {
+                    Group: group,
+                },
+                Post: null,
+            },
+        ],
+    };
+    await new Promise((resolve, reject) => {
+        emailsUtils.filterNotificationForDelivery(notification, user, "point_activity", "A subject", (error) => {
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+    assert.equal(warnings.length, 1);
+    assert.equal(warnings[0].message, "Redis suppression lookup failed, continuing delivery");
+    assert.equal(warnings[0].metadata.userId, 42);
+    assert.equal(queuedEmails.length, 1);
+    assert.equal(queuedEmails[0][0], "send-one-email");
+    assert.equal(redisSetExCalls.length, 1);
+});

package/tests/emails_utils.test.d.cts

@@ -0,0 +1 @@
+export {};