@yrpri/api 9.0.103 → 9.0.104

package/dist/scripts/processCsvForTranslationAndToxicity.js

@@ -1,125 +0,0 @@
-const models = require('../models/index.cjs');
-const async = require('async');
-const _ = require('lodash');
-const fs = require('fs');
-const { Translate } = require('@google-cloud/translate').v2;
-const inFile = "/home/robert/Scratch/delpoll.csv"; //process.argv[3];
-const outFile = "/home/robert/Scratch/outFileDelPollForToxicity.csv"; //process.argv[4];
-const fromLang = "is";
-const toLang = "en";
-let outFileContent = "";
-const Perspective = require('perspective-api-client');
-const parse = require('csv-parse/lib/sync');
-let perspectiveApi;
-if (process.env.GOOGLE_PERSPECTIVE_API_KEY) {
-    perspectiveApi = new Perspective({ apiKey: process.env.GOOGLE_PERSPECTIVE_API_KEY });
-}
-const getToxicityScoreForText = (text, doNotStore, callback) => {
-    console.log("getToxicityScoreForText starting", { text, doNotStore });
-    if (text) {
-        perspectiveApi.analyze(text, { doNotStore, attributes: [
-                'TOXICITY', 'SEVERE_TOXICITY', 'IDENTITY_ATTACK',
-                'THREAT', 'INSULT', 'PROFANITY', 'SEXUALLY_EXPLICIT',
-                'FLIRTATION'
-            ] }).then(result => {
-            console.log("getToxicityScoreForText results");
-            callback(null, result);
-        }).catch(error => {
-            if (error && error.stack && error.stack.indexOf("ResponseError: Attribute") > -1) {
-                console.warn("getToxicityScoreForText warning", { error });
-                callback(error);
-            }
-            else {
-                console.error("getToxicityScoreForText error", { error });
-                callback(error);
-            }
-        });
-    }
-    else {
-        callback("No text for toxicity score");
-    }
-};
-const clean = (text) => {
-    //console.log("Before: "+ text);
-    var newText = text.replace('"', "'").replace('\n', '').replace('\r', '').replace(/(\r\n|\n|\r)/gm, "").replace(/"/gm, "'").replace(/,/, ';').trim();
-    //console.log("After:" + newText);
-    return newText.replace(/´/g, '');
-};
-const getTranslatedString = (contentToTranslate, callback) => {
-    const translateAPI = new Translate({
-        credentials: JSON.parse(process.env.GOOGLE_APPLICATION_CREDENTIALS_JSON)
-    });
-    translateAPI.translate(contentToTranslate, toLang)
-        .then((results) => {
-        const translationResults = results[1];
-        if (translationResults && translationResults.data
-            && translationResults.data.translations &&
-            translationResults.data.translations.length > 0) {
-            const translation = translationResults.data.translations[0];
-            callback(null, translation);
-        }
-        else {
-            callback("No translations");
-        }
-    }).catch((error) => {
-        callback(error);
-    });
-};
-const addHeader = () => {
-    outFileContent += "Original,English,Toxicity,Severe Toxicity,Profanity,Sexually Explicit,Flirtation,Insult\n";
-};
-const addRow = (original, english, toxicity, severeToxicity, profanity, sexuallyExplicit, flirtation, threat, insult) => {
-    outFileContent += '"' + original + '"' + ',"' + english + '","' + toxicity + '",' + '"' + severeToxicity + '",';
-    outFileContent += '"' + profanity + '"' + ',"' + sexuallyExplicit + '","' + flirtation + '",' + '"' + insult + '",' + '\n';
-};
-const file = fs.readFileSync(inFile, 'utf-8');
-let records = parse(file, {
-    columns: false,
-    skip_empty_lines: false
-});
-addHeader();
-async.forEachSeries(records, (line, eachCallback) => {
-    getTranslatedString(line[0], (error, translatedString) => {
-        if (translatedString && translatedString.translatedText) {
-            getToxicityScoreForText(translatedString.translatedText, true, (error, score) => {
-                if (error) {
-                    console.error(error);
-                    addRow(line, translatedString.translatedText, "error");
-                }
-                else {
-                    const toxicityScore = score.attributeScores["TOXICITY"].summaryScore.value;
-                    const severeToxicityScore = score.attributeScores["SEVERE_TOXICITY"].summaryScore.value;
-                    const profanityScore = score.attributeScores["PROFANITY"].summaryScore.value;
-                    const sexuallyExplicitScore = score.attributeScores["SEXUALLY_EXPLICIT"].summaryScore.value;
-                    const flirtationScore = score.attributeScores["FLIRTATION"].summaryScore.value;
-                    const threatScore = score.attributeScores["THREAT"].summaryScore.value;
-                    const insultScore = score.attributeScores["INSULT"].summaryScore.value;
-                    addRow(line, translatedString.translatedText, toxicityScore, severeToxicityScore, profanityScore, sexuallyExplicitScore, flirtationScore, threatScore, insultScore);
-                }
-                eachCallback();
-            });
-        }
-        else {
-            addRow(line, "NOT TRANSLATED");
-            eachCallback();
-        }
-    });
-}, (error) => {
-    if (error) {
-        console.error(error);
-        process.exit();
-    }
-    else {
-        fs.writeFile(outFile, outFileContent, function (error) {
-            if (error) {
-                console.error(error);
-                process.exit();
-            }
-            else {
-                console.log("The file was saved!");
-                process.exit();
-            }
-        });
-    }
-});
-export {};

package/dist/scripts/recount/recount_recursive_communities.js

@@ -1,125 +0,0 @@
-const models = require('../../models/index.cjs');
-let users = [];
-let postCount = 0;
-let pointCount = 0;
-let groupCount = 0;
-const processedCommunityIds = [];
-const processedGroupIds = [];
-function mergeArrays(...arrays) {
-    let jointArray = [];
-    arrays.forEach(array => {
-        jointArray = [...jointArray, ...array];
-    });
-    return [...new Set([...jointArray])];
-}
-const countCommunities = async (communityId, options) => {
-    return await new Promise((resolve, reject) => {
-        models.Community.findOne({
-            where: {
-                id: communityId
-            },
-            attributes: ['id', 'name'],
-            include: [
-                {
-                    model: models.Group,
-                    attributes: ['id', 'name', 'community_id', 'configuration', 'counter_posts', 'counter_points', 'is_group_folder'],
-                    where: {
-                        is_group_folder: false
-                    },
-                    include: [
-                        {
-                            model: models.User,
-                            as: 'GroupUsers',
-                            attributes: ['id'],
-                            required: false
-                        }
-                    ],
-                    required: false
-                }
-            ]
-        }).then(async (community) => {
-            if (community) {
-                const linksToVisit = [];
-                for (let i = 0; i < community.Groups.length; i++) {
-                    const group = community.Groups[i];
-                    if (group.configuration.actAsLinkToCommunityId) {
-                        if (false && processedCommunityIds.indexOf(group.configuration.actAsLinkToCommunityId) > -1) {
-                            console.warn(`DUPLICATE LINK Group Id ${group.id} in Community Id ${group.community_id} to Community Id ${group.configuration.actAsLinkToCommunityId}`);
-                        }
-                        else {
-                            processedCommunityIds.push(group.configuration.actAsLinkToCommunityId);
-                            if (processedGroupIds.indexOf(group.id) > -1) {
-                                console.warn(`DUPLICATE GROUP Group Id ${group.id} in Community Id ${group.community_id} to Community Id ${group.configuration.actAsLinkToCommunityId}`);
-                            }
-                            else {
-                                processedGroupIds.push(group.id);
-                                await countCommunities(group.configuration.actAsLinkToCommunityId, options);
-                            }
-                        }
-                    }
-                    else {
-                        users = mergeArrays(users, group.GroupUsers.map((u) => u.id));
-                        postCount += group.counter_posts;
-                        pointCount += group.counter_points;
-                        groupCount += 1;
-                        const random = Math.random();
-                        if (random < 0.05) {
-                            //console.log(`Groups ${groupCount} Posts ${postCount} Points ${pointCount} Users ${users.length}`)
-                        }
-                    }
-                }
-                resolve();
-            }
-            else {
-                console.error("Cant find community " + communityId);
-                resolve();
-            }
-        }).catch(error => {
-            reject(error);
-        });
-    });
-};
-const recountCommunityRecursive = async (communityId, options) => {
-    return await new Promise((resolve, reject) => {
-        models.Community.findOne({
-            where: {
-                id: communityId
-            },
-            attributes: ['name', 'id']
-        }).then(async (community) => {
-            try {
-                await countCommunities(communityId, options);
-                community.counter_users = users.length;
-                community.counter_posts = postCount;
-                community.counter_points = pointCount;
-                community.counter_groups = groupCount;
-                await community.save();
-                console.log(`Save community ${users.length} ${postCount} ${pointCount}`);
-                resolve();
-            }
-            catch (error) {
-                reject(error);
-            }
-        }).catch(error => {
-            reject(error);
-        });
-    });
-};
-(async function () {
-    const communities = await models.Community.findAll({
-        where: {
-            "configuration.recalculateCountersRecursively": true
-        },
-        attributes: ['id', 'configuration']
-    });
-    for (let i = 0; i < communities.length; i++) {
-        console.log("Processing community: " + communities[i].id);
-        users = [];
-        postCount = 0;
-        pointCount = 0;
-        groupCount = 0;
-        await recountCommunityRecursive(communities[i].id, {});
-    }
-    process.exit();
-})();
-export {};

package/dist/scripts/recountALLCommunityGroupCounts.js

@@ -1,37 +0,0 @@
-const models = require('../models/index.cjs');
-const async = require('async');
-const _ = require('lodash');
-const fs = require('fs');
-const request = require('request');
-models.Community.findAll({
-    attributes: ['id', 'counter_groups']
-}).then(allCommunities => {
-    async.forEachLimit(allCommunities, 5, (community, forEachCallback) => {
-        models.Group.count({
-            where: {
-                community_id: community.id
-            }
-        }).then(count => {
-            community.update({
-                counter_groups: count
-            }).then(() => {
-                console.log(`Updated Community ${community.id} to ${count}`);
-                forEachCallback();
-            }).catch(error => {
-                forEachCallback(error);
-            });
-        }).catch(error => {
-            forEachCallback(error);
-        });
-    }, (error) => {
-        console.log(error);
-        process.exit();
-    });
-}).catch(error => {
-    if (error)
-        console.error(error);
-    else
-        console.log(done);
-    process.exit();
-});
-export {};

package/dist/scripts/recountAll.js

@@ -1,97 +0,0 @@
-var models = require('../models/index.cjs');
-var async = require('async');
-var _ = require('lodash');
-var recountOnePost = function (postId, done) {
-    var endorsementsCount;
-    var oppositionCount;
-    var pointCount;
-    async.parallel([
-        function (parallelCallback) {
-            models.Endorsement.count({
-                where: {
-                    value: 1,
-                    post_id: postId
-                }
-            }).then(function (count) {
-                endorsementsCount = count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        },
-        function (parallelCallback) {
-            models.Endorsement.count({
-                where: {
-                    value: -1,
-                    post_id: postId
-                }
-            }).then(function (count) {
-                oppositionCount = count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        },
-        function (parallelCallback) {
-            models.Point.count({
-                where: {
-                    $or: [
-                        { value: -1 },
-                        { value: 1 }
-                    ],
-                    post_id: postId
-                }
-            }).then(function (count) {
-                pointCount = count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        }
-    ], function (error) {
-        if (error) {
-            done(error);
-        }
-        else {
-            models.Post.findOne({
-                where: {
-                    id: postId
-                },
-                attributes: ['id', 'counter_endorsements_up', 'counter_endorsements_down', 'counter_points']
-            }).then(function (post) {
-                post.counter_points = pointCount;
-                post.counter_endorsements_up = endorsementsCount;
-                post.counter_endorsements_down = oppositionCount;
-                post.save().then(function (results) {
-                    console.log("Results: " + results);
-                    done();
-                });
-            }).catch(function (error) {
-                done(error);
-            });
-        }
-    });
-};
-var recountAllPosts = function (done) {
-    models.Post.findAll({
-        attribute: ['id']
-    }).then(function (postIds) {
-        async.eachSeries(postIds, function (post, forEachCallback) {
-            recountOnePost(post.id, forEachCallback);
-        });
-    });
-};
-async.series([
-    function (seriesCallback) {
-        recountAllPosts(seriesCallback);
-    }
-], function (error) {
-    if (error) {
-        console.error(error);
-    }
-    else {
-        console.log("Done counting all");
-    }
-    process.exit();
-});
-export {};

package/dist/scripts/recountCommunitesFromUrl.js

@@ -1,58 +0,0 @@
-const models = require('../models/index.cjs');
-const async = require('async');
-const _ = require('lodash');
-const fs = require('fs');
-const request = require('request');
-const recountCommunity = require('../utils/recount_utils.cjs').recountCommunity;
-/*
-const urlToConfig = "https://yrpri-eu-direct-assets.s3-eu-west-1.amazonaws.com/WBMoveIdeas160221.csv"//process.argv[1];
-const urlToAddAddFront = "https://kyrgyz-aris.yrpri.org/"; // process.argv[2];
-*/
-const urlToConfig = process.argv[2];
-const urlToAddAddFront = process.argv[3];
-let config;
-let finalTargetOutput = '';
-async.series([
-    (seriesCallback) => {
-        const options = {
-            url: urlToConfig,
-        };
-        request.get(options, (error, content) => {
-            if (content && content.statusCode != 200) {
-                seriesCallback(content.statusCode);
-            }
-            else if (content) {
-                config = content.body;
-                seriesCallback();
-            }
-            else {
-                seriesCallback("No content");
-            }
-        });
-    },
-    (seriesCallback) => {
-        let index = 0;
-        async.forEachSeries(config.split('\r\n'), (configLine, forEachCallback) => {
-            const splitLine = configLine.split(",");
-            if (index === 0 || !configLine || configLine.length < 3 || !splitLine || splitLine.length !== 1) {
-                index += 1;
-                forEachCallback();
-            }
-            else {
-                index += 1;
-                const communityId = splitLine[0];
-                finalTargetOutput += urlToAddAddFront + "community/" + communityId + "\n\n";
-                recountCommunity(communityId, forEachCallback);
-            }
-        }, error => {
-            seriesCallback(error);
-        });
-    },
-], error => {
-    if (error)
-        console.error(error);
-    console.log("All done recounting");
-    console.log(finalTargetOutput);
-    process.exit();
-});
-export {};

package/dist/scripts/recountCommunity.js

@@ -1,19 +0,0 @@
-const models = require('../models/index.cjs');
-const async = require('async');
-const _ = require('lodash');
-const fs = require('fs');
-const request = require('request');
-const recountCommunity = require('../utils/recount_utils.cjs').recountCommunity;
-/*
-const urlToConfig = "https://yrpri-eu-direct-assets.s3-eu-west-1.amazonaws.com/WBMoveIdeas160221.csv"//process.argv[1];
-const urlToAddAddFront = "https://kyrgyz-aris.yrpri.org/"; // process.argv[2];
-*/
-const communityId = process.argv[2];
-recountCommunity(communityId, error => {
-    if (error) {
-        console.error(error);
-    }
-    console.log("Done recounting");
-    process.exit();
-});
-export {};

package/dist/scripts/recountGroup.js

@@ -1,218 +0,0 @@
-var models = require('../models/index.cjs');
-var async = require('async');
-var _ = require('lodash');
-const groupId = process.argv[2];
-let masterPostCount = 0;
-let masterPointCount = 0;
-let masterEndorsementCount = 0;
-let masterQualityCount = 0;
-const recountOnePoint = (pointId, done) => {
-    var endorsementsCount;
-    var oppositionCount;
-    let revisionsCount;
-    async.parallel([
-        (parallelCallback) => {
-            models.PointQuality.count({
-                where: {
-                    value: 1,
-                    point_id: pointId
-                }
-            }).then(function (count) {
-                endorsementsCount = count;
-                masterQualityCount += count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        },
-        (parallelCallback) => {
-            models.PointQuality.count({
-                where: {
-                    value: -1,
-                    point_id: pointId
-                }
-            }).then(function (count) {
-                oppositionCount = count;
-                masterQualityCount += count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        },
-        (parallelCallback) => {
-            models.PointRevision.count({
-                where: {
-                    point_id: pointId
-                }
-            }).then(function (count) {
-                pointCount = count;
-                revisionsCount = count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        }
-    ], function (error) {
-        if (error) {
-            done(error);
-        }
-        else {
-            models.Point.findOne({
-                where: {
-                    id: pointId
-                },
-                attributes: ['id', 'counter_revisions', 'counter_quality_up', 'counter_quality_down']
-            }).then(function (point) {
-                point.counter_revisions = revisionsCount;
-                point.counter_quality_up = endorsementsCount;
-                point.counter_quality_down = oppositionCount;
-                point.save().then(() => {
-                    done();
-                });
-            }).catch(error => {
-                done(error);
-            });
-        }
-    });
-};
-const recountOnePost = (postId, done) => {
-    var endorsementsCount;
-    var oppositionCount;
-    var pointCount;
-    async.parallel([
-        (parallelCallback) => {
-            models.Endorsement.count({
-                where: {
-                    value: 1,
-                    post_id: postId
-                }
-            }).then(function (count) {
-                endorsementsCount = count;
-                masterEndorsementCount += count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        },
-        (parallelCallback) => {
-            models.Endorsement.count({
-                where: {
-                    value: -1,
-                    post_id: postId
-                }
-            }).then(function (count) {
-                oppositionCount = count;
-                masterEndorsementCount += count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        },
-        (parallelCallback) => {
-            models.Point.count({
-                where: {
-                    $or: [
-                        { value: -1 },
-                        { value: 1 }
-                    ],
-                    post_id: postId
-                }
-            }).then(function (count) {
-                pointCount = count;
-                masterPointCount += count;
-                parallelCallback();
-            }).catch(function (error) {
-                parallelCallback(error);
-            });
-        }
-    ], function (error) {
-        if (error) {
-            done(error);
-        }
-        else {
-            models.Post.findOne({
-                where: {
-                    id: postId
-                },
-                attributes: ['id', 'counter_endorsements_up', 'counter_endorsements_down', 'counter_points']
-            }).then(function (post) {
-                post.counter_points = pointCount;
-                post.counter_endorsements_up = endorsementsCount;
-                post.counter_endorsements_down = oppositionCount;
-                post.save().then(function (results) {
-                    console.log("Results: " + results);
-                    done();
-                });
-            }).catch(function (error) {
-                done(error);
-            });
-        }
-    });
-};
-const recountGroupPosts = (groupId, done) => {
-    models.Post.findAll({
-        where: {
-            group_id: groupId
-        },
-        attribute: ['id']
-    }).then(function (posts) {
-        masterPostCount = posts.length;
-        async.eachSeries(posts, (post, forEachCallback) => {
-            recountOnePost(post.id, (error) => {
-                recountPostPoints(post.id, forEachCallback);
-            });
-        }, error => {
-            done(error);
-        });
-    });
-};
-const recountPostPoints = (postId, done) => {
-    models.Point.findAll({
-        where: {
-            post_id: postId
-        },
-        attribute: ['id']
-    }).then(function (points) {
-        async.eachSeries(points, (point, forEachCallback) => {
-            recountOnePoint(point.id, forEachCallback);
-        }, error => {
-            done(error);
-        });
-    });
-};
-recountGroupPosts(groupId, error => {
-    if (error) {
-        console.error(error);
-        process.exit();
-    }
-    else {
-        models.Group.findOne({
-            where: {
-                id: groupId
-            },
-            include: [
-                {
-                    model: models.User,
-                    as: 'GroupUsers',
-                    attributes: ['id']
-                }
-            ],
-            attributes: ['id', 'counter_posts', 'counter_points', 'counter_users']
-        }).then(group => {
-            group.counter_posts = masterPostCount;
-            group.counter_points = masterPointCount;
-            group.counter_users = group.GroupUsers.length;
-            group.save().then(() => {
-                console.log(`Done recounting group ${groupId}`);
-                process.exit();
-            }).catch(error => {
-                console.error(error);
-                process.exit();
-            });
-        }).catch(error => {
-            console.error(error);
-            process.exit();
-        });
-    }
-});
-export {};