@yrpri/api 9.0.101 → 9.0.103

package/dist/scripts/processCsvForTranslationAndToxicity.js

@@ -0,0 +1,125 @@
+const models = require('../models/index.cjs');
+const async = require('async');
+const _ = require('lodash');
+const fs = require('fs');
+const { Translate } = require('@google-cloud/translate').v2;
+const inFile = "/home/robert/Scratch/delpoll.csv"; //process.argv[3];
+const outFile = "/home/robert/Scratch/outFileDelPollForToxicity.csv"; //process.argv[4];
+const fromLang = "is";
+const toLang = "en";
+let outFileContent = "";
+const Perspective = require('perspective-api-client');
+const parse = require('csv-parse/lib/sync');
+let perspectiveApi;
+if (process.env.GOOGLE_PERSPECTIVE_API_KEY) {
+    perspectiveApi = new Perspective({ apiKey: process.env.GOOGLE_PERSPECTIVE_API_KEY });
+}
+const getToxicityScoreForText = (text, doNotStore, callback) => {
+    console.log("getToxicityScoreForText starting", { text, doNotStore });
+    if (text) {
+        perspectiveApi.analyze(text, { doNotStore, attributes: [
+                'TOXICITY', 'SEVERE_TOXICITY', 'IDENTITY_ATTACK',
+                'THREAT', 'INSULT', 'PROFANITY', 'SEXUALLY_EXPLICIT',
+                'FLIRTATION'
+            ] }).then(result => {
+            console.log("getToxicityScoreForText results");
+            callback(null, result);
+        }).catch(error => {
+            if (error && error.stack && error.stack.indexOf("ResponseError: Attribute") > -1) {
+                console.warn("getToxicityScoreForText warning", { error });
+                callback(error);
+            }
+            else {
+                console.error("getToxicityScoreForText error", { error });
+                callback(error);
+            }
+        });
+    }
+    else {
+        callback("No text for toxicity score");
+    }
+};
+const clean = (text) => {
+    //console.log("Before: "+ text);
+    var newText = text.replace('"', "'").replace('\n', '').replace('\r', '').replace(/(\r\n|\n|\r)/gm, "").replace(/"/gm, "'").replace(/,/, ';').trim();
+    //console.log("After:" + newText);
+    return newText.replace(/´/g, '');
+};
+const getTranslatedString = (contentToTranslate, callback) => {
+    const translateAPI = new Translate({
+        credentials: JSON.parse(process.env.GOOGLE_APPLICATION_CREDENTIALS_JSON)
+    });
+    translateAPI.translate(contentToTranslate, toLang)
+        .then((results) => {
+        const translationResults = results[1];
+        if (translationResults && translationResults.data
+            && translationResults.data.translations &&
+            translationResults.data.translations.length > 0) {
+            const translation = translationResults.data.translations[0];
+            callback(null, translation);
+        }
+        else {
+            callback("No translations");
+        }
+    }).catch((error) => {
+        callback(error);
+    });
+};
+const addHeader = () => {
+    outFileContent += "Original,English,Toxicity,Severe Toxicity,Profanity,Sexually Explicit,Flirtation,Insult\n";
+};
+const addRow = (original, english, toxicity, severeToxicity, profanity, sexuallyExplicit, flirtation, threat, insult) => {
+    outFileContent += '"' + original + '"' + ',"' + english + '","' + toxicity + '",' + '"' + severeToxicity + '",';
+    outFileContent += '"' + profanity + '"' + ',"' + sexuallyExplicit + '","' + flirtation + '",' + '"' + insult + '",' + '\n';
+};
+const file = fs.readFileSync(inFile, 'utf-8');
+let records = parse(file, {
+    columns: false,
+    skip_empty_lines: false
+});
+addHeader();
+async.forEachSeries(records, (line, eachCallback) => {
+    getTranslatedString(line[0], (error, translatedString) => {
+        if (translatedString && translatedString.translatedText) {
+            getToxicityScoreForText(translatedString.translatedText, true, (error, score) => {
+                if (error) {
+                    console.error(error);
+                    addRow(line, translatedString.translatedText, "error");
+                }
+                else {
+                    const toxicityScore = score.attributeScores["TOXICITY"].summaryScore.value;
+                    const severeToxicityScore = score.attributeScores["SEVERE_TOXICITY"].summaryScore.value;
+                    const profanityScore = score.attributeScores["PROFANITY"].summaryScore.value;
+                    const sexuallyExplicitScore = score.attributeScores["SEXUALLY_EXPLICIT"].summaryScore.value;
+                    const flirtationScore = score.attributeScores["FLIRTATION"].summaryScore.value;
+                    const threatScore = score.attributeScores["THREAT"].summaryScore.value;
+                    const insultScore = score.attributeScores["INSULT"].summaryScore.value;
+                    addRow(line, translatedString.translatedText, toxicityScore, severeToxicityScore, profanityScore, sexuallyExplicitScore, flirtationScore, threatScore, insultScore);
+                }
+                eachCallback();
+            });
+        }
+        else {
+            addRow(line, "NOT TRANSLATED");
+            eachCallback();
+        }
+    });
+}, (error) => {
+    if (error) {
+        console.error(error);
+        process.exit();
+    }
+    else {
+        fs.writeFile(outFile, outFileContent, function (error) {
+            if (error) {
+                console.error(error);
+                process.exit();
+            }
+            else {
+                console.log("The file was saved!");
+                process.exit();
+            }
+        });
+    }
+});
+export {};

package/dist/scripts/recount/recount_recursive_communities.js

@@ -0,0 +1,125 @@
+const models = require('../../models/index.cjs');
+let users = [];
+let postCount = 0;
+let pointCount = 0;
+let groupCount = 0;
+const processedCommunityIds = [];
+const processedGroupIds = [];
+function mergeArrays(...arrays) {
+    let jointArray = [];
+    arrays.forEach(array => {
+        jointArray = [...jointArray, ...array];
+    });
+    return [...new Set([...jointArray])];
+}
+const countCommunities = async (communityId, options) => {
+    return await new Promise((resolve, reject) => {
+        models.Community.findOne({
+            where: {
+                id: communityId
+            },
+            attributes: ['id', 'name'],
+            include: [
+                {
+                    model: models.Group,
+                    attributes: ['id', 'name', 'community_id', 'configuration', 'counter_posts', 'counter_points', 'is_group_folder'],
+                    where: {
+                        is_group_folder: false
+                    },
+                    include: [
+                        {
+                            model: models.User,
+                            as: 'GroupUsers',
+                            attributes: ['id'],
+                            required: false
+                        }
+                    ],
+                    required: false
+                }
+            ]
+        }).then(async (community) => {
+            if (community) {
+                const linksToVisit = [];
+                for (let i = 0; i < community.Groups.length; i++) {
+                    const group = community.Groups[i];
+                    if (group.configuration.actAsLinkToCommunityId) {
+                        if (false && processedCommunityIds.indexOf(group.configuration.actAsLinkToCommunityId) > -1) {
+                            console.warn(`DUPLICATE LINK Group Id ${group.id} in Community Id ${group.community_id} to Community Id ${group.configuration.actAsLinkToCommunityId}`);
+                        }
+                        else {
+                            processedCommunityIds.push(group.configuration.actAsLinkToCommunityId);
+                            if (processedGroupIds.indexOf(group.id) > -1) {
+                                console.warn(`DUPLICATE GROUP Group Id ${group.id} in Community Id ${group.community_id} to Community Id ${group.configuration.actAsLinkToCommunityId}`);
+                            }
+                            else {
+                                processedGroupIds.push(group.id);
+                                await countCommunities(group.configuration.actAsLinkToCommunityId, options);
+                            }
+                        }
+                    }
+                    else {
+                        users = mergeArrays(users, group.GroupUsers.map((u) => u.id));
+                        postCount += group.counter_posts;
+                        pointCount += group.counter_points;
+                        groupCount += 1;
+                        const random = Math.random();
+                        if (random < 0.05) {
+                            //console.log(`Groups ${groupCount} Posts ${postCount} Points ${pointCount} Users ${users.length}`)
+                        }
+                    }
+                }
+                resolve();
+            }
+            else {
+                console.error("Cant find community " + communityId);
+                resolve();
+            }
+        }).catch(error => {
+            reject(error);
+        });
+    });
+};
+const recountCommunityRecursive = async (communityId, options) => {
+    return await new Promise((resolve, reject) => {
+        models.Community.findOne({
+            where: {
+                id: communityId
+            },
+            attributes: ['name', 'id']
+        }).then(async (community) => {
+            try {
+                await countCommunities(communityId, options);
+                community.counter_users = users.length;
+                community.counter_posts = postCount;
+                community.counter_points = pointCount;
+                community.counter_groups = groupCount;
+                await community.save();
+                console.log(`Save community ${users.length} ${postCount} ${pointCount}`);
+                resolve();
+            }
+            catch (error) {
+                reject(error);
+            }
+        }).catch(error => {
+            reject(error);
+        });
+    });
+};
+(async function () {
+    const communities = await models.Community.findAll({
+        where: {
+            "configuration.recalculateCountersRecursively": true
+        },
+        attributes: ['id', 'configuration']
+    });
+    for (let i = 0; i < communities.length; i++) {
+        console.log("Processing community: " + communities[i].id);
+        users = [];
+        postCount = 0;
+        pointCount = 0;
+        groupCount = 0;
+        await recountCommunityRecursive(communities[i].id, {});
+    }
+    process.exit();
+})();
+export {};

package/dist/scripts/recountALLCommunityGroupCounts.js

@@ -0,0 +1,37 @@
+const models = require('../models/index.cjs');
+const async = require('async');
+const _ = require('lodash');
+const fs = require('fs');
+const request = require('request');
+models.Community.findAll({
+    attributes: ['id', 'counter_groups']
+}).then(allCommunities => {
+    async.forEachLimit(allCommunities, 5, (community, forEachCallback) => {
+        models.Group.count({
+            where: {
+                community_id: community.id
+            }
+        }).then(count => {
+            community.update({
+                counter_groups: count
+            }).then(() => {
+                console.log(`Updated Community ${community.id} to ${count}`);
+                forEachCallback();
+            }).catch(error => {
+                forEachCallback(error);
+            });
+        }).catch(error => {
+            forEachCallback(error);
+        });
+    }, (error) => {
+        console.log(error);
+        process.exit();
+    });
+}).catch(error => {
+    if (error)
+        console.error(error);
+    else
+        console.log(done);
+    process.exit();
+});
+export {};

package/dist/scripts/recountAll.js

@@ -0,0 +1,97 @@
+var models = require('../models/index.cjs');
+var async = require('async');
+var _ = require('lodash');
+var recountOnePost = function (postId, done) {
+    var endorsementsCount;
+    var oppositionCount;
+    var pointCount;
+    async.parallel([
+        function (parallelCallback) {
+            models.Endorsement.count({
+                where: {
+                    value: 1,
+                    post_id: postId
+                }
+            }).then(function (count) {
+                endorsementsCount = count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        },
+        function (parallelCallback) {
+            models.Endorsement.count({
+                where: {
+                    value: -1,
+                    post_id: postId
+                }
+            }).then(function (count) {
+                oppositionCount = count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        },
+        function (parallelCallback) {
+            models.Point.count({
+                where: {
+                    $or: [
+                        { value: -1 },
+                        { value: 1 }
+                    ],
+                    post_id: postId
+                }
+            }).then(function (count) {
+                pointCount = count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        }
+    ], function (error) {
+        if (error) {
+            done(error);
+        }
+        else {
+            models.Post.findOne({
+                where: {
+                    id: postId
+                },
+                attributes: ['id', 'counter_endorsements_up', 'counter_endorsements_down', 'counter_points']
+            }).then(function (post) {
+                post.counter_points = pointCount;
+                post.counter_endorsements_up = endorsementsCount;
+                post.counter_endorsements_down = oppositionCount;
+                post.save().then(function (results) {
+                    console.log("Results: " + results);
+                    done();
+                });
+            }).catch(function (error) {
+                done(error);
+            });
+        }
+    });
+};
+var recountAllPosts = function (done) {
+    models.Post.findAll({
+        attribute: ['id']
+    }).then(function (postIds) {
+        async.eachSeries(postIds, function (post, forEachCallback) {
+            recountOnePost(post.id, forEachCallback);
+        });
+    });
+};
+async.series([
+    function (seriesCallback) {
+        recountAllPosts(seriesCallback);
+    }
+], function (error) {
+    if (error) {
+        console.error(error);
+    }
+    else {
+        console.log("Done counting all");
+    }
+    process.exit();
+});
+export {};

package/dist/scripts/recountCommunitesFromUrl.js

@@ -0,0 +1,58 @@
+const models = require('../models/index.cjs');
+const async = require('async');
+const _ = require('lodash');
+const fs = require('fs');
+const request = require('request');
+const recountCommunity = require('../utils/recount_utils.cjs').recountCommunity;
+/*
+const urlToConfig = "https://yrpri-eu-direct-assets.s3-eu-west-1.amazonaws.com/WBMoveIdeas160221.csv"//process.argv[1];
+const urlToAddAddFront = "https://kyrgyz-aris.yrpri.org/"; // process.argv[2];
+*/
+const urlToConfig = process.argv[2];
+const urlToAddAddFront = process.argv[3];
+let config;
+let finalTargetOutput = '';
+async.series([
+    (seriesCallback) => {
+        const options = {
+            url: urlToConfig,
+        };
+        request.get(options, (error, content) => {
+            if (content && content.statusCode != 200) {
+                seriesCallback(content.statusCode);
+            }
+            else if (content) {
+                config = content.body;
+                seriesCallback();
+            }
+            else {
+                seriesCallback("No content");
+            }
+        });
+    },
+    (seriesCallback) => {
+        let index = 0;
+        async.forEachSeries(config.split('\r\n'), (configLine, forEachCallback) => {
+            const splitLine = configLine.split(",");
+            if (index === 0 || !configLine || configLine.length < 3 || !splitLine || splitLine.length !== 1) {
+                index += 1;
+                forEachCallback();
+            }
+            else {
+                index += 1;
+                const communityId = splitLine[0];
+                finalTargetOutput += urlToAddAddFront + "community/" + communityId + "\n\n";
+                recountCommunity(communityId, forEachCallback);
+            }
+        }, error => {
+            seriesCallback(error);
+        });
+    },
+], error => {
+    if (error)
+        console.error(error);
+    console.log("All done recounting");
+    console.log(finalTargetOutput);
+    process.exit();
+});
+export {};

package/dist/scripts/recountCommunity.js

@@ -0,0 +1,19 @@
+const models = require('../models/index.cjs');
+const async = require('async');
+const _ = require('lodash');
+const fs = require('fs');
+const request = require('request');
+const recountCommunity = require('../utils/recount_utils.cjs').recountCommunity;
+/*
+const urlToConfig = "https://yrpri-eu-direct-assets.s3-eu-west-1.amazonaws.com/WBMoveIdeas160221.csv"//process.argv[1];
+const urlToAddAddFront = "https://kyrgyz-aris.yrpri.org/"; // process.argv[2];
+*/
+const communityId = process.argv[2];
+recountCommunity(communityId, error => {
+    if (error) {
+        console.error(error);
+    }
+    console.log("Done recounting");
+    process.exit();
+});
+export {};

package/dist/scripts/recountGroup.js

@@ -0,0 +1,218 @@
+var models = require('../models/index.cjs');
+var async = require('async');
+var _ = require('lodash');
+const groupId = process.argv[2];
+let masterPostCount = 0;
+let masterPointCount = 0;
+let masterEndorsementCount = 0;
+let masterQualityCount = 0;
+const recountOnePoint = (pointId, done) => {
+    var endorsementsCount;
+    var oppositionCount;
+    let revisionsCount;
+    async.parallel([
+        (parallelCallback) => {
+            models.PointQuality.count({
+                where: {
+                    value: 1,
+                    point_id: pointId
+                }
+            }).then(function (count) {
+                endorsementsCount = count;
+                masterQualityCount += count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        },
+        (parallelCallback) => {
+            models.PointQuality.count({
+                where: {
+                    value: -1,
+                    point_id: pointId
+                }
+            }).then(function (count) {
+                oppositionCount = count;
+                masterQualityCount += count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        },
+        (parallelCallback) => {
+            models.PointRevision.count({
+                where: {
+                    point_id: pointId
+                }
+            }).then(function (count) {
+                pointCount = count;
+                revisionsCount = count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        }
+    ], function (error) {
+        if (error) {
+            done(error);
+        }
+        else {
+            models.Point.findOne({
+                where: {
+                    id: pointId
+                },
+                attributes: ['id', 'counter_revisions', 'counter_quality_up', 'counter_quality_down']
+            }).then(function (point) {
+                point.counter_revisions = revisionsCount;
+                point.counter_quality_up = endorsementsCount;
+                point.counter_quality_down = oppositionCount;
+                point.save().then(() => {
+                    done();
+                });
+            }).catch(error => {
+                done(error);
+            });
+        }
+    });
+};
+const recountOnePost = (postId, done) => {
+    var endorsementsCount;
+    var oppositionCount;
+    var pointCount;
+    async.parallel([
+        (parallelCallback) => {
+            models.Endorsement.count({
+                where: {
+                    value: 1,
+                    post_id: postId
+                }
+            }).then(function (count) {
+                endorsementsCount = count;
+                masterEndorsementCount += count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        },
+        (parallelCallback) => {
+            models.Endorsement.count({
+                where: {
+                    value: -1,
+                    post_id: postId
+                }
+            }).then(function (count) {
+                oppositionCount = count;
+                masterEndorsementCount += count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        },
+        (parallelCallback) => {
+            models.Point.count({
+                where: {
+                    $or: [
+                        { value: -1 },
+                        { value: 1 }
+                    ],
+                    post_id: postId
+                }
+            }).then(function (count) {
+                pointCount = count;
+                masterPointCount += count;
+                parallelCallback();
+            }).catch(function (error) {
+                parallelCallback(error);
+            });
+        }
+    ], function (error) {
+        if (error) {
+            done(error);
+        }
+        else {
+            models.Post.findOne({
+                where: {
+                    id: postId
+                },
+                attributes: ['id', 'counter_endorsements_up', 'counter_endorsements_down', 'counter_points']
+            }).then(function (post) {
+                post.counter_points = pointCount;
+                post.counter_endorsements_up = endorsementsCount;
+                post.counter_endorsements_down = oppositionCount;
+                post.save().then(function (results) {
+                    console.log("Results: " + results);
+                    done();
+                });
+            }).catch(function (error) {
+                done(error);
+            });
+        }
+    });
+};
+const recountGroupPosts = (groupId, done) => {
+    models.Post.findAll({
+        where: {
+            group_id: groupId
+        },
+        attribute: ['id']
+    }).then(function (posts) {
+        masterPostCount = posts.length;
+        async.eachSeries(posts, (post, forEachCallback) => {
+            recountOnePost(post.id, (error) => {
+                recountPostPoints(post.id, forEachCallback);
+            });
+        }, error => {
+            done(error);
+        });
+    });
+};
+const recountPostPoints = (postId, done) => {
+    models.Point.findAll({
+        where: {
+            post_id: postId
+        },
+        attribute: ['id']
+    }).then(function (points) {
+        async.eachSeries(points, (point, forEachCallback) => {
+            recountOnePoint(point.id, forEachCallback);
+        }, error => {
+            done(error);
+        });
+    });
+};
+recountGroupPosts(groupId, error => {
+    if (error) {
+        console.error(error);
+        process.exit();
+    }
+    else {
+        models.Group.findOne({
+            where: {
+                id: groupId
+            },
+            include: [
+                {
+                    model: models.User,
+                    as: 'GroupUsers',
+                    attributes: ['id']
+                }
+            ],
+            attributes: ['id', 'counter_posts', 'counter_points', 'counter_users']
+        }).then(group => {
+            group.counter_posts = masterPostCount;
+            group.counter_points = masterPointCount;
+            group.counter_users = group.GroupUsers.length;
+            group.save().then(() => {
+                console.log(`Done recounting group ${groupId}`);
+                process.exit();
+            }).catch(error => {
+                console.error(error);
+                process.exit();
+            });
+        }).catch(error => {
+            console.error(error);
+            process.exit();
+        });
+    }
+});
+export {};