@yrpri/api 9.0.231 → 9.0.233

package/services/engine/moderation/fraud/FraudDeleteRatings.cjs

@@ -4,12 +4,69 @@ const FraudDeleteEndorsements = require('./FraudDeleteEndorsements.cjs');
 const models = require("../../../../models/index.cjs");
 //TODO: Change to native JS instead of lodash
 class FraudDeleteRatings extends FraudDeleteEndorsements {
-    async destroyChunkItems(items) {
-        return this.destroyChunkItemsByModel(models.Rating, items);
+    getAllItemsExceptOne(items) {
+        if (items.length === 1 && this.getAllowedSingleDelete()) {
+            return items;
+        }
+        const itemsByTypeIndex = _.groupBy(items, item => {
+            return item.type_index === undefined || item.type_index === null
+                ? "default"
+                : item.type_index;
+        });
+        let itemsToDelete = [];
+        _.each(itemsByTypeIndex, innerItems => {
+            itemsToDelete = itemsToDelete.concat(super.getAllItemsExceptOne(innerItems));
+        });
+        return itemsToDelete;
+    }
+    async destroyChunkItems(items, transaction) {
+        return this.destroyChunkItemsByModel(models.Rating, items, transaction);
     }
     async getItemsById() {
-        const itemsContainer = await this.getModelItemsById(models.Rating);
-        return itemsContainer.itemsToAnalyse;
+        return await new Promise(async (resolve, reject) => {
+            try {
+                const items = await models.Rating.findAll({
+                    where: {
+                        id: {
+                            $in: this.job.internal_data.idsToDelete
+                        }
+                    },
+                    attributes: ["id", "created_at", "value", "type_index", "post_id", "user_id", "user_agent", "ip_address", "data"],
+                    include: [
+                        {
+                            model: models.User,
+                            attributes: ['id', 'email'],
+                        },
+                        {
+                            model: models.Post,
+                            attributes: ['id', 'name'],
+                            include: [
+                                {
+                                    model: models.Group,
+                                    attributes: ['id'],
+                                    include: [
+                                        {
+                                            model: models.Community,
+                                            attributes: [],
+                                            where: {
+                                                id: this.workPackage.communityId
+                                            }
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                });
+                const itemsToAnalyse = _.sortBy(items, function (item) {
+                    return [item.post_id, item.type_index, item.user_agent];
+                });
+                resolve(itemsToAnalyse);
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
     }
 }
 module.exports = FraudDeleteRatings;

package/services/engine/moderation/fraud/FraudGetBase.cjs

@@ -26,6 +26,21 @@ class FraudGetBase extends FraudBase {
             }
         });
     }
+    getUserEmail(item) {
+        return item.User && item.User.email ? item.User.email : "";
+    }
+    getUserName(item) {
+        return item.User && item.User.name ? item.User.name : "";
+    }
+    getPostName(item) {
+        return item.Post && item.Post.name ? item.Post.name : "";
+    }
+    getPointQualityPostName(item) {
+        return item.Point && item.Point.Post && item.Point.Post.name ? item.Point.Post.name : "";
+    }
+    getItemName(item) {
+        return item && item.name ? item.name : "";
+    }
     customCompress() {
         const flatData = [];
         _.forEach(this.dataToProcess, item => {
@@ -71,43 +86,52 @@ class FraudGetBase extends FraudBase {
                 cDoneUserAgents[item.user_agent] = true;
                 outData.cUserAgents.push(item.user_agent);
             }
-            if (!cDoneEmails[item.User.email]) {
-                cDoneEmails[item.User.email] = true;
-                outData.cEmails.push(item.User.email);
+            const userEmail = this.getUserEmail(item);
+            const userName = this.getUserName(item);
+            item.User = item.User || {};
+            if (!cDoneEmails[userEmail]) {
+                cDoneEmails[userEmail] = true;
+                outData.cEmails.push(userEmail);
             }
-            if (!cDoneNames[item.User.name]) {
-                cDoneNames[item.User.name] = true;
-                outData.cNames.push(item.User.name);
+            if (!cDoneNames[userName]) {
+                cDoneNames[userName] = true;
+                outData.cNames.push(userName);
             }
             if (this.workPackage.collectionType === "endorsements" ||
                 this.workPackage.collectionType === "ratings" ||
                 this.workPackage.collectionType === "points") {
-                if (!cDonePostNames[item.Post.name]) {
-                    cDonePostNames[item.Post.name] = true;
-                    outData.cPostNames.push(item.Post.name);
+                const postName = this.getPostName(item);
+                item.Post = item.Post || {};
+                if (!cDonePostNames[postName]) {
+                    cDonePostNames[postName] = true;
+                    outData.cPostNames.push(postName);
                 }
-                item.Post.name = outData.cPostNames.indexOf(item.Post.name);
+                item.Post.name = outData.cPostNames.indexOf(postName);
             }
             if (this.workPackage.collectionType === "posts") {
-                if (!cDonePostNames[item.name]) {
-                    cDonePostNames[item.name] = true;
-                    outData.cPostNames.push(item.name);
+                const postName = this.getItemName(item);
+                if (!cDonePostNames[postName]) {
+                    cDonePostNames[postName] = true;
+                    outData.cPostNames.push(postName);
                 }
-                item.name = outData.cPostNames.indexOf(item.name);
+                item.name = outData.cPostNames.indexOf(postName);
             }
             if (this.workPackage.collectionType === "pointQualities") {
-                if (!cDonePostNames[item.Point.Post.name]) {
-                    cDonePostNames[item.Point.Post.name] = true;
-                    outData.cPostNames.push(item.Point.Post.name);
+                const postName = this.getPointQualityPostName(item);
+                item.Point = item.Point || {};
+                item.Point.Post = item.Point.Post || {};
+                if (!cDonePostNames[postName]) {
+                    cDonePostNames[postName] = true;
+                    outData.cPostNames.push(postName);
                 }
-                item.Point.Post.name = outData.cPostNames.indexOf(item.Point.Post.name);
+                item.Point.Post.name = outData.cPostNames.indexOf(postName);
             }
             item.dataValues.backgroundColor = outData.cBackgroundColors.indexOf(item.dataValues.backgroundColor);
             item.dataValues.confidenceScoreSort = parseInt(item.dataValues.confidenceScore.replace("%", ''));
             item.ip_address = outData.cIpAddresses.indexOf(item.ip_address);
             item.user_agent = outData.cUserAgents.indexOf(item.user_agent);
-            item.User.email = outData.cEmails.indexOf(item.User.email);
-            item.User.name = outData.cEmails.indexOf(item.User.name);
+            item.User.email = outData.cEmails.indexOf(userEmail);
+            item.User.name = outData.cNames.indexOf(userName);
             outData.items.push(item);
         });
         this.dataToProcess = outData;

package/services/engine/moderation/fraud/FraudGetBase.d.cts

@@ -2,6 +2,11 @@ export = FraudGetBase;
 declare class FraudGetBase extends FraudBase {
     formatTime(): void;
     setBackgroundColorsFromKey(): void;
+    getUserEmail(item: any): any;
+    getUserName(item: any): any;
+    getPostName(item: any): any;
+    getPointQualityPostName(item: any): any;
+    getItemName(item: any): any;
     customCompress(): void;
     processAndGetFraudItems(): Promise<any>;
 }

package/services/engine/moderation/fraud/FraudGetEndorsements.cjs

@@ -39,19 +39,7 @@ class FraudGetEndorsements extends FraudGetBase {
                 const itemsToAnalyse = _.sortBy(items, function (item) {
                     return [item.post_id, item.user_agent];
                 });
-                let groupConfiguration;
-                if (getGroup && items.length > 0) {
-                    const group = await models.Group.findOne({
-                        where: {
-                            id: itemsToAnalyse[0].Post.Group.id
-                        },
-                        attributes: ['configuration']
-                    });
-                    if (group) {
-                        groupConfiguration = group.configuration;
-                    }
-                }
-                resolve({ itemsToAnalyse, groupConfiguration });
+                resolve({ itemsToAnalyse });
             }
             catch (error) {
                 reject(error);
@@ -66,6 +54,9 @@ class FraudGetEndorsements extends FraudGetBase {
         let topItems = this.setupTopItems(items);
         const postIds = this.getPostIdsFromItems(topItems);
         const postCount = _.uniq(postIds).length;
+        if (this.isDebugFraudDetectionCountAll()) {
+            return this.getDebugTopItems(topItems);
+        }
         if (type === "byIpUserAgentPostId") {
             let out = [];
             _.each(topItems, function (item) {

package/services/engine/moderation/fraud/FraudGetEndorsements.d.cts

@@ -2,6 +2,6 @@ export = FraudGetEndorsements;
 declare class FraudGetEndorsements extends FraudGetBase {
     getAllModelItems(model: any, getGroup: any): Promise<any>;
     getAllItems(): Promise<any>;
-    getTopItems(items: any, type: any): any[];
+    getTopItems(items: any, type: any): any;
 }
 import FraudGetBase = require("./FraudGetBase.cjs");

package/services/engine/moderation/fraud/FraudGetPointQualities.cjs

@@ -65,6 +65,9 @@ class FraudGetPointQualities extends FraudGetBase {
         let topItems = this.setupTopItems(items);
         const pointIds = this.getPointIdsFromItems(topItems);
         const pointCount = _.uniq(pointIds).length;
+        if (this.isDebugFraudDetectionCountAll()) {
+            return this.getDebugTopItems(topItems);
+        }
         if (type === "byIpUserAgentPointId") {
             let out = [];
             _.each(topItems, function (item) {

package/services/engine/moderation/fraud/FraudGetPointQualities.d.cts

@@ -2,6 +2,6 @@ export = FraudGetPointQualities;
 declare class FraudGetPointQualities extends FraudGetBase {
     getAllItems(): Promise<any>;
     getPointIdsFromItems(topItems: any): any[];
-    getTopItems(items: any, type: any): any[];
+    getTopItems(items: any, type: any): any;
 }
 import FraudGetBase = require("./FraudGetBase.cjs");

package/services/engine/moderation/fraud/FraudGetPoints.cjs

@@ -51,6 +51,9 @@ class FraudGetPointQualities extends FraudGetBase {
         const postIds = this.getPostIdsFromItems(topItems);
         const postCount = _.uniq(postIds).length;
         const pointMultiplier = 10;
+        if (this.isDebugFraudDetectionCountAll()) {
+            return this.getDebugTopItems(topItems);
+        }
         if (type === "byIpUserAgentPostId") {
             let out = [];
             _.each(topItems, function (item) {

package/services/engine/moderation/fraud/FraudGetPoints.d.cts

@@ -1,6 +1,6 @@
 export = FraudGetPointQualities;
 declare class FraudGetPointQualities extends FraudGetBase {
     getAllItems(): Promise<any>;
-    getTopItems(items: any, type: any): any[];
+    getTopItems(items: any, type: any): any;
 }
 import FraudGetBase = require("./FraudGetBase.cjs");

package/services/engine/moderation/fraud/FraudGetPosts.cjs

@@ -3,7 +3,7 @@ const _ = require("lodash");
 const FraudGetBase = require('./FraudGetBase.cjs');
 const models = require("../../../../models/index.cjs");
 const log = require("../../../../utils/logger.cjs");
-class FraudGetPoints extends FraudGetBase {
+class FraudGetPosts extends FraudGetBase {
     async getAllItems() {
         return await new Promise(async (resolve, reject) => {
             try {
@@ -41,23 +41,24 @@ class FraudGetPoints extends FraudGetBase {
     }
     getTopItems(items, type) {
         let topItems = this.setupTopItems(items);
-        const postIds = this.getPostIdsFromItems(topItems);
-        const postCount = _.uniq(postIds).length;
         const pointMultiplier = 10;
+        if (this.isDebugFraudDetectionCountAll()) {
+            return this.getDebugTopItems(topItems);
+        }
         if (type === "byIpFingerprint") {
             let out = [];
             _.each(topItems, function (item) {
-                if ((item.count / postCount) > pointMultiplier) {
-                    if ((item.count / postCount) > 5 * pointMultiplier) {
+                if (item.count > pointMultiplier) {
+                    if (item.count > 5 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 70);
                     }
-                    else if ((item.count / postCount) > 4 * pointMultiplier) {
+                    else if (item.count > 4 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 60);
                     }
-                    else if ((item.count / postCount) > 3 * pointMultiplier) {
+                    else if (item.count > 3 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 40);
                     }
-                    else if ((item.count / postCount) > 2 * pointMultiplier) {
+                    else if (item.count > 2 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 20);
                     }
                     else {
@@ -74,23 +75,23 @@ class FraudGetPoints extends FraudGetBase {
         else if (type === "byIpAddress") {
             let out = [];
             _.each(topItems, function (item) {
-                if ((item.count / postCount) > pointMultiplier) {
-                    if ((item.count / postCount) > 100 * pointMultiplier) {
+                if (item.count > pointMultiplier) {
+                    if (item.count > 100 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 70);
                     }
-                    else if ((item.count / postCount) > 50 * pointMultiplier) {
+                    else if (item.count > 50 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 65);
                     }
-                    else if ((item.count / postCount) > 25 * pointMultiplier) {
+                    else if (item.count > 25 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 40);
                     }
-                    else if ((item.count / postCount) > 10 * pointMultiplier) {
+                    else if (item.count > 10 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 35);
                     }
-                    else if ((item.count / postCount) > 5 * pointMultiplier) {
+                    else if (item.count > 5 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 20);
                     }
-                    else if ((item.count / postCount) > 2 * pointMultiplier) {
+                    else if (item.count > 2 * pointMultiplier) {
                         this.setWeightedConfidenceScore(item.items, 15);
                     }
                     else {
@@ -107,4 +108,4 @@ class FraudGetPoints extends FraudGetBase {
         }
     }
 }
-module.exports = FraudGetPoints;
+module.exports = FraudGetPosts;

package/services/engine/moderation/fraud/FraudGetPosts.d.cts

@@ -1,6 +1,6 @@
-export = FraudGetPoints;
-declare class FraudGetPoints extends FraudGetBase {
+export = FraudGetPosts;
+declare class FraudGetPosts extends FraudGetBase {
     getAllItems(): Promise<any>;
-    getTopItems(items: any, type: any): any[];
+    getTopItems(items: any, type: any): any;
 }
 import FraudGetBase = require("./FraudGetBase.cjs");

package/services/engine/moderation/fraud/FraudGetRatings.cjs

@@ -6,38 +6,67 @@ const log = require("../../../../utils/logger.cjs");
 class FraudGetRatings extends FraudGetEndorsements {
     constructor(workPackage) {
         super(workPackage);
-        this.groupConfiguration = null;
+        this.ratingDimensionCountsByGroupId = {};
     }
     async getAllItems() {
         const itemsContainer = await this.getAllModelItems(models.Rating, true);
-        this.groupConfiguration = itemsContainer.groupConfiguration;
+        this.ratingDimensionCountsByGroupId = await this.getRatingDimensionCountsByGroupId(itemsContainer.itemsToAnalyse);
         return itemsContainer.itemsToAnalyse;
     }
+    async getRatingDimensionCountsByGroupId(items) {
+        const groupIds = _.uniq(items.map(item => {
+            return item.Post && item.Post.Group && item.Post.Group.id;
+        }).filter(groupId => groupId));
+        if (groupIds.length === 0) {
+            return {};
+        }
+        const groups = await models.Group.findAll({
+            where: {
+                id: groupIds
+            },
+            attributes: ['id', 'configuration']
+        });
+        return groups.reduce((countsByGroupId, group) => {
+            const customRatings = group.configuration && group.configuration.customRatings;
+            countsByGroupId[group.id] =
+                customRatings && typeof customRatings === "object"
+                    ? Math.max(Object.keys(customRatings).length, 1)
+                    : 1;
+            return countsByGroupId;
+        }, {});
+    }
+    getRatingDimensionsForItem(item) {
+        const groupId = item.Post && item.Post.Group && item.Post.Group.id;
+        return this.ratingDimensionCountsByGroupId[groupId] || 1;
+    }
+    getNormalizedRatingCount(items) {
+        return _.sumBy(items, item => 1 / this.getRatingDimensionsForItem(item));
+    }
     getTopItems(items, type) {
         let topItems = this.setupTopItems(items);
         const postIds = this.getPostIdsFromItems(topItems);
-        const postCount = _.uniq(postIds).length;
-        let ratingsCount = 10000000;
-        if (this.groupConfiguration) {
-            ratingsCount = Object.keys(this.groupConfiguration.customRatings).length;
+        const postCount = Math.max(_.uniq(postIds).length, 1);
+        if (this.isDebugFraudDetectionCountAll()) {
+            return this.getDebugTopItems(topItems);
         }
         if (type === "byIpUserAgentPostId") {
             let out = [];
             _.each(topItems, function (item) {
-                if ((item.count / ratingsCount) > 1) {
-                    if ((item.count / ratingsCount) > 10) {
+                const ratingRatio = this.getNormalizedRatingCount(item.items);
+                if (ratingRatio > 1) {
+                    if (ratingRatio > 10) {
                         this.setWeightedConfidenceScore(item.items, 95);
                     }
-                    else if ((item.count / ratingsCount) > 5) {
+                    else if (ratingRatio > 5) {
                         this.setWeightedConfidenceScore(item.items, 90);
                     }
-                    else if ((item.count / ratingsCount) > 4) {
+                    else if (ratingRatio > 4) {
                         this.setWeightedConfidenceScore(item.items, 85);
                     }
-                    else if ((item.count / ratingsCount) > 3) {
+                    else if (ratingRatio > 3) {
                         this.setWeightedConfidenceScore(item.items, 80);
                     }
-                    else if ((item.count / ratingsCount) > 2) {
+                    else if (ratingRatio > 2) {
                         this.setWeightedConfidenceScore(item.items, 75);
                     }
                     else {
@@ -51,17 +80,18 @@ class FraudGetRatings extends FraudGetEndorsements {
         else if (type === "byIpFingerprint") {
             let out = [];
             _.each(topItems, function (item) {
-                if ((item.count / postCount / ratingsCount) > 1) {
-                    if ((item.count / postCount / ratingsCount) > 5) {
+                const ratingRatio = this.getNormalizedRatingCount(item.items) / postCount;
+                if (ratingRatio > 1) {
+                    if (ratingRatio > 5) {
                         this.setWeightedConfidenceScore(item.items, 99);
                     }
-                    else if ((item.count / postCount / ratingsCount) > 4) {
+                    else if (ratingRatio > 4) {
                         this.setWeightedConfidenceScore(item.items, 95);
                     }
-                    else if ((item.count / postCount / ratingsCount) > 3) {
+                    else if (ratingRatio > 3) {
                         this.setWeightedConfidenceScore(item.items, 90);
                     }
-                    else if ((item.count / postCount / ratingsCount) > 2) {
+                    else if (ratingRatio > 2) {
                         this.setWeightedConfidenceScore(item.items, 85);
                     }
                     else {
@@ -78,20 +108,21 @@ class FraudGetRatings extends FraudGetEndorsements {
         else if (type === "byIpFingerprintPostId") {
             let out = [];
             _.each(topItems, function (item) {
-                if ((item.count / ratingsCount) > 1) {
-                    if ((item.count / ratingsCount) > 10) {
+                const ratingRatio = this.getNormalizedRatingCount(item.items);
+                if (ratingRatio > 1) {
+                    if (ratingRatio > 10) {
                         this.setWeightedConfidenceScore(item.items, 100);
                     }
-                    else if ((item.count / ratingsCount) > 5) {
+                    else if (ratingRatio > 5) {
                         this.setWeightedConfidenceScore(item.items, 98);
                     }
-                    else if ((item.count / ratingsCount) > 4) {
+                    else if (ratingRatio > 4) {
                         this.setWeightedConfidenceScore(item.items, 95);
                     }
-                    else if ((item.count / ratingsCount) > 3) {
+                    else if (ratingRatio > 3) {
                         this.setWeightedConfidenceScore(item.items, 90);
                     }
-                    else if ((item.count / ratingsCount) > 2) {
+                    else if (ratingRatio > 2) {
                         this.setWeightedConfidenceScore(item.items, 85);
                     }
                     else {
@@ -105,23 +136,24 @@ class FraudGetRatings extends FraudGetEndorsements {
         else if (type === "byIpAddress") {
             let out = [];
             _.each(topItems, function (item) {
-                if ((item.count / postCount / ratingsCount) > 1) {
-                    if ((item.count / postCount / ratingsCount) > 100) {
+                const ratingRatio = this.getNormalizedRatingCount(item.items) / postCount;
+                if (ratingRatio > 1) {
+                    if (ratingRatio > 100) {
                         this.setWeightedConfidenceScore(item.items, 90);
                     }
-                    else if ((item.count / postCount / ratingsCount) > 50) {
+                    else if (ratingRatio > 50) {
                         this.setWeightedConfidenceScore(item.items, 85);
                     }
-                    else if ((item.count / postCount / ratingsCount) > 25) {
+                    else if (ratingRatio > 25) {
                         this.setWeightedConfidenceScore(item.items, 80);
                     }
-                    else if ((item.count / postCount / ratingsCount) > 10) {
+                    else if (ratingRatio > 10) {
                         this.setWeightedConfidenceScore(item.items, 75);
                     }
-                    else if ((item.count / postCount / ratingsCount) > 5) {
+                    else if (ratingRatio > 5) {
                         this.setWeightedConfidenceScore(item.items, 70);
                     }
-                    else if ((item.count / postCount / ratingsCount) > 2) {
+                    else if (ratingRatio > 2) {
                         this.setWeightedConfidenceScore(item.items, 65);
                     }
                     else {

package/services/engine/moderation/fraud/FraudGetRatings.d.cts

@@ -1,5 +1,8 @@
 export = FraudGetRatings;
 declare class FraudGetRatings extends FraudGetEndorsements {
-    groupConfiguration: any;
+    ratingDimensionCountsByGroupId: {};
+    getRatingDimensionCountsByGroupId(items: any): Promise<{}>;
+    getRatingDimensionsForItem(item: any): any;
+    getNormalizedRatingCount(items: any): number;
 }
 import FraudGetEndorsements = require("./FraudGetEndorsements.cjs");