@yrpri/api 9.0.101 → 9.0.103

package/dist/scripts/displayAuthorForPost.js

@@ -0,0 +1,16 @@
+var models = require('../models/index.cjs');
+var async = require('async');
+var _ = require('lodash');
+var postId = process.argv[2];
+models.Post.findOne({
+    where: {
+        id: postId
+    },
+    include: [
+        models.User
+    ]
+}).then(function (post) {
+    console.log(post.User.name + " " + post.User.email);
+    process.exit();
+});
+export {};

package/dist/scripts/endorsementFraudDetection/analyseEndorsementsForCommunity.js

@@ -0,0 +1,183 @@
+var models = require('../../models/index.cjs');
+var async = require('async');
+var _ = require('lodash');
+var communityId = process.argv[2];
+var groupId = process.argv[3];
+var endorsementsToAnalyse;
+var csvOut = "Method,E Id,E Value,Date,Browser Id,Fingerprint Id,IP Address,User Id,User Email,Post Id,Post Name,User Agent\n";
+var writeItemToCsv = function (item) {
+    let browserId = "";
+    let browserFingerprint = "";
+    if (item.data) {
+        browserId = item.data.browserId;
+        browserFingerprint = item.data.browserFingerprint;
+    }
+    return "," + item.id + "," + item.value + "," + item.created_at +
+        "," + browserId + "," + browserFingerprint +
+        "," + item.ip_address + "," + item.user_id + "," + item.User.email + "," +
+        item.post_id + ',"' + item.Post.name + '","' + item.user_agent + '"\n';
+};
+var writeItemsToCsv = function (header, items) {
+    csvOut += '"' + header + '",,,,,,\n';
+    _.forEach(items, function (item) {
+        csvOut += '"' + item.key + ' (' + item.count + ")" + '",,,,,,\n';
+        _.forEach(item.items, function (innerItem) {
+            csvOut += writeItemToCsv(innerItem);
+        });
+    });
+};
+var getTopItems = function (items, max) {
+    if (!max)
+        max = 25;
+    var topItems = [];
+    _.each(items, function (items, key) {
+        topItems.push({ key: key, count: items.length, items: items });
+    });
+    topItems = _.sortBy(topItems, function (item) {
+        return -item.count;
+    });
+    if (max === -1) {
+        let out = [];
+        _.each(topItems, function (item) {
+            if (item.count > 1) {
+                out.push(item);
+            }
+        });
+        return out;
+    }
+    else if (max === -2) {
+        let out = [];
+        _.each(topItems, function (item) {
+            if (item.count > 10) {
+                out.push(item);
+            }
+        });
+        return out;
+    }
+    else {
+        return _.take(topItems, max);
+    }
+};
+async.series([
+    // Get all Community Endorsements
+    function (seriesCallback) {
+        if (communityId && !groupId) {
+            models.Endorsement.findAll({
+                attributes: ["id", "created_at", "value", "post_id", "user_id", "user_agent", "ip_address", "data"],
+                include: [
+                    {
+                        model: models.User,
+                        attributes: ['id', 'name', 'email'],
+                    },
+                    {
+                        model: models.Post,
+                        attributes: ['id', 'name', 'group_id'],
+                        include: [
+                            {
+                                model: models.Group,
+                                attributes: ['id', 'name'],
+                                include: [
+                                    {
+                                        model: models.Community,
+                                        attributes: ['id', 'name'],
+                                        where: {
+                                            id: communityId
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                ]
+            }).then(function (endorsements) {
+                endorsementsToAnalyse = endorsements;
+                endorsementsToAnalyse = _.sortBy(endorsementsToAnalyse, function (item) {
+                    return [item.post_id, item.user_agent];
+                });
+                seriesCallback();
+            });
+        }
+        else {
+            seriesCallback();
+        }
+    },
+    // Get all Group Endorsements
+    function (seriesCallback) {
+        if (groupId) {
+            models.Endorsement.findAll({
+                attributes: ["id", "post_id", "value", "user_id", "user_agent", "ip_address", "data"],
+                include: [
+                    {
+                        model: models.User,
+                        attributes: ['id', 'name', 'email'],
+                    },
+                    {
+                        model: models.Post,
+                        attributes: ['id', 'name', 'group_id'],
+                        include: [
+                            {
+                                model: models.Group,
+                                where: {
+                                    id: groupId
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }).then(function (endorsements) {
+                endorsementsToAnalyse = endorsements;
+                seriesCallback();
+            });
+        }
+        else {
+            seriesCallback();
+        }
+    },
+    // Top 10 IPs Unique User Agents + Post Ids
+    function (seriesCallback) {
+        var groupedByIPs = _.groupBy(endorsementsToAnalyse, function (endorsement) {
+            return endorsement.ip_address + ":" + endorsement.post_id + ":" + endorsement.user_agent;
+        });
+        writeItemsToCsv("Top votes from IP, User agent, Post Id", getTopItems(groupedByIPs, -1));
+        seriesCallback();
+    },
+    // Top 10 IPs
+    function (seriesCallback) {
+        var groupedByIPs = _.groupBy(endorsementsToAnalyse, function (endorsement) {
+            return endorsement.ip_address;
+        });
+        writeItemsToCsv("Top votes from IPs", getTopItems(groupedByIPs, -2));
+        seriesCallback();
+    },
+    // Top 10 IPs Unique User Agents
+    /*function (seriesCallback) {
+      var groupedByIPs = _.groupBy(endorsementsToAnalyse, function (endorsement) {
+        return endorsement.ip_address+":"+endorsement.user_agent;
+      });
+      writeItemsToCsv("Top votes from IP & User agent", getTopItems(groupedByIPs));
+      seriesCallback();
+    },
+    // Top 10 IPs Unique User Agents + User Ids
+    function (seriesCallback) {
+      var groupedByIPs = _.groupBy(endorsementsToAnalyse, function (endorsement) {
+        return endorsement.ip_address+":"+endorsement.user_id+":"+endorsement.user_agent;
+      });
+      writeItemsToCsv("Top votes from IP User agent & User Id", getTopItems(groupedByIPs));
+      seriesCallback();
+    },
+    // Top 10 IPs Unique User Agents + User Ids + Post Ids
+    function (seriesCallback) {
+      var groupedByIPs = _.groupBy(endorsementsToAnalyse, function (endorsement) {
+        return endorsement.ip_address+":"+endorsement.user_id+":"+endorsement.post_id+":"+endorsement.user_agent;
+      });
+      writeItemsToCsv("Top votes from IP, User agent, User Id, Post Id", getTopItems(groupedByIPs));
+      seriesCallback();
+    }*/
+], function (error) {
+    if (error) {
+        console.error(error);
+    }
+    console.log(csvOut);
+    process.exit();
+});
+export {};

package/dist/scripts/endorsementFraudDetection/bulkDeleteDuplicateEndorsmentsFromUrl.js

@@ -0,0 +1,208 @@
+const models = require('../../models/index.cjs');
+const async = require('async');
+const _ = require('lodash');
+const fs = require('fs');
+const request = require('request');
+const recountCommunity = require('../../utils/recount_utils.cjs').recountCommunity;
+const recountPost = require('../../utils/recount_utils.cjs').recountPost;
+const communityId = process.argv[2];
+const urlToConfig = process.argv[3];
+const allowDeletingSingles = process.argv[4];
+const allItems = [];
+let deletedEndorsments = 0;
+const postsToRecount = [];
+console.log("Starting...");
+function CSVToArray(strData, strDelimiter) {
+    // Check to see if the delimiter is defined. If not,
+    // then default to comma.
+    strDelimiter = (strDelimiter || ",");
+    // Create a regular expression to parse the CSV values.
+    var objPattern = new RegExp((
+    // Delimiters.
+    "(\\" + strDelimiter + "|\\r?\\n|\\r|^)" +
+        // Quoted fields.
+        "(?:\"([^\"]*(?:\"\"[^\"]*)*)\"|" +
+        // Standard fields.
+        "([^\"\\" + strDelimiter + "\\r\\n]*))"), "gi");
+    // Create an array to hold our data. Give the array
+    // a default empty first row.
+    var arrData = [[]];
+    // Create an array to hold our individual pattern
+    // matching groups.
+    var arrMatches = null;
+    // Keep looping over the regular expression matches
+    // until we can no longer find a match.
+    while (arrMatches = objPattern.exec(strData)) {
+        // Get the delimiter that was found.
+        var strMatchedDelimiter = arrMatches[1];
+        // Check to see if the given delimiter has a length
+        // (is not the start of string) and if it matches
+        // field delimiter. If id does not, then we know
+        // that this delimiter is a row delimiter.
+        if (strMatchedDelimiter.length &&
+            strMatchedDelimiter !== strDelimiter) {
+            // Since we have reached a new row of data,
+            // add an empty row to our data array.
+            arrData.push([]);
+        }
+        var strMatchedValue;
+        // Now that we have our delimiter out of the way,
+        // let's check to see which kind of value we
+        // captured (quoted or unquoted).
+        if (arrMatches[2]) {
+            // We found a quoted value. When we capture
+            // this value, unescape any double quotes.
+            strMatchedValue = arrMatches[2].replace(new RegExp("\"\"", "g"), "\"");
+        }
+        else {
+            // We found a non-quoted value.
+            strMatchedValue = arrMatches[3];
+        }
+        // Now that we have our value string, let's add
+        // it to the data array.
+        arrData[arrData.length - 1].push(strMatchedValue);
+    }
+    // Return the parsed data.
+    return (arrData);
+}
+const processItemsToDestroy = (itemsToDestroy, callback) => {
+    async.forEachSeries(itemsToDestroy, (item, forEachItemCallback) => {
+        models.Endorsement.findOne({
+            where: {
+                id: item.endorsementId
+            },
+            attributes: ['id']
+        }).then(endorsement => {
+            if (endorsement) {
+                endorsement.deleted = true;
+                endorsement.save().then(() => {
+                    deletedEndorsments++;
+                    forEachItemCallback();
+                }).catch(error => {
+                    forEachItemCallback(error);
+                });
+            }
+            else {
+                console.warn("Endorsement not found: " + item.endorsementId);
+                forEachItemCallback();
+            }
+        }).catch(error => {
+            forEachItemCallback(error);
+        });
+    }, error => {
+        if (error) {
+            callback(error);
+        }
+        else {
+            if (postsToRecount.indexOf(itemsToDestroy[0].postId) == -1) {
+                postsToRecount.push(itemsToDestroy[0].postId);
+            }
+            callback();
+        }
+    });
+};
+const getAllItemsExceptOne = (items) => {
+    if (items.length === 1 && allowDeletingSingles) {
+        return items;
+    }
+    else {
+        const sortedItems = _.sortBy(items, function (item) {
+            return item.date;
+        });
+        const finalItems = [];
+        let foundEmail = false;
+        for (let i = 0; i < sortedItems.length; i++) {
+            if (!foundEmail && sortedItems[i].userEmail.indexOf("_anonymous@citizens.i") === -1) {
+                foundEmail = true;
+            }
+            else {
+                finalItems.push(sortedItems[i]);
+            }
+        }
+        if (items.length === finalItems.length) {
+            finalItems.pop();
+        }
+        return finalItems;
+    }
+};
+let config;
+let parsedConfig;
+async.series([
+    (seriesCallback) => {
+        const options = {
+            url: urlToConfig,
+        };
+        console.log(`Get ${urlToConfig}`);
+        request.get(options, (error, content) => {
+            if (content && content.statusCode != 200) {
+                seriesCallback(content.statusCode);
+            }
+            else if (content) {
+                config = content.body;
+                parsedConfig = CSVToArray(config);
+                console.log("Got content");
+                seriesCallback();
+            }
+            else {
+                seriesCallback("No content");
+            }
+        });
+    },
+    (seriesCallback) => {
+        for (let i = 0; i < parsedConfig.length; i++) {
+            const splitLine = parsedConfig[i];
+            if (splitLine[0] === "" && splitLine[1] !== "") {
+                allItems.push({
+                    endorsementId: splitLine[1],
+                    endorsementValue: splitLine[2],
+                    date: splitLine[3],
+                    browserId: splitLine[4],
+                    browserFingerprint: splitLine[5],
+                    ipAddress: splitLine[6],
+                    userId: splitLine[7],
+                    userEmail: splitLine[8],
+                    postId: splitLine[9],
+                    postName: splitLine[10],
+                    userAgent: splitLine[11]
+                });
+                console.log(`Will delete vote for ${splitLine[10]} from ${splitLine[8]}`);
+            }
+        }
+        seriesCallback();
+    },
+    (seriesCallback) => {
+        const chunks = _.groupBy(allItems, function (endorsement) {
+            return endorsement.ipAddress + ":" + endorsement.postId + ":" + endorsement.userAgent;
+        });
+        console.log("Processing chunks");
+        async.forEachSeries(chunks, (items, forEachChunkCallback) => {
+            const itemsToDestroy = getAllItemsExceptOne(items);
+            if (itemsToDestroy.length > 0) {
+                processItemsToDestroy(itemsToDestroy, forEachChunkCallback);
+            }
+            else {
+                console.warn("Items length == 0 and no allowDeletingSingles, skipping");
+                forEachChunkCallback();
+            }
+        }, error => {
+            seriesCallback(error);
+        });
+    },
+    (seriesCallback) => {
+        async.forEachSeries(postsToRecount, (postId, forEachPostCallback) => {
+            recountPost(postId, forEachPostCallback);
+        }, error => {
+            seriesCallback(error);
+        });
+    },
+    (seriesCallback) => {
+        recountCommunity(communityId, seriesCallback);
+    }
+], error => {
+    if (error)
+        console.error(error);
+    else
+        console.log(`Deleted ${deletedEndorsments} endorsements`);
+    process.exit();
+});
+export {};

package/dist/scripts/exportAllStatusChanges.js

@@ -0,0 +1,36 @@
+var models = require('../models/index.cjs');
+var async = require('async');
+var domainId = process.argv[2];
+models.PostStatusChange.findAll({
+    include: [
+        {
+            model: models.Post,
+            attributes: ['id'],
+            required: true,
+            include: [
+                {
+                    model: models.Group,
+                    attributes: ['id'],
+                    where: {
+                        access: models.Group.ACCESS_PUBLIC
+                    },
+                    required: true
+                }
+            ]
+        }
+    ]
+}).then(function (updates) {
+    async.eachSeries(updates, function (update, seriesCallback) {
+        if (update.content) {
+            console.log("------------------------------------------ STATUS CHANGE ID " + update.id + " ------------------------------------------");
+            console.log("\n");
+            console.log(update.content);
+            console.log("\n");
+        }
+        seriesCallback();
+    }, function () {
+        process.exit();
+        console.log("Done");
+    });
+});
+export {};

package/dist/scripts/exportClientAcitivity.js

@@ -0,0 +1,36 @@
+var models = require('../models/index.cjs');
+var async = require('async');
+var _ = require('lodash');
+var moment = require('moment');
+const type = "domain"; //process.argv[2];
+const containerId = "422"; //= process.argv[3];
+const fromDate = moment("14/11/2019", "DD/MM/YYYY").toISOString(); //process.argv[4];
+const endDate = moment("30/12/2019", "DD/MM/YYYY").toISOString(); //process.argv[5];
+const typeFilter = "completed"; //process.argv[6];
+const actionFilter = "newPost"; //process.argv[7];
+const oldData = true; // process.argv[7] ? true ; false;
+if (oldData) {
+    models.AcActivity.findAll({
+        where: {
+            $and: [
+                {
+                    created_at: {
+                        $gt: fromDate
+                    }
+                },
+                {
+                    created_at: {
+                        $lt: endDate
+                    }
+                }
+            ]
+        },
+        attributes: ['object', 'actor', 'type', 'sub_type']
+    }).then(function (actitities) {
+        process.exit();
+    });
+}
+else {
+    process.exit();
+}
+export {};

package/dist/scripts/exportEndorsementsForCommunity.js

@@ -0,0 +1,79 @@
+var models = require('../models/index.cjs');
+var async = require('async');
+var _ = require('lodash');
+var fs = require('fs');
+var communityId = process.argv[2];
+var outFile = process.argv[3];
+var clean = function (text) {
+    //console.log("Before: "+ text);
+    if (text) {
+        var newText = text.replace('"', "'").replace('\n', '').replace('\r', '').replace(/(\r\n|\n|\r)/gm, "").replace(/"/gm, "'").replace(',', ';').trim();
+        //console.log("After:" + newText);
+        return newText.replace(/´/g, '');
+    }
+    else {
+        return "empty";
+    }
+};
+var getType = function (value) {
+    if (value > 0) {
+        return "for";
+    }
+    else if (value < 0) {
+        return "against";
+    }
+    else {
+        return "comment/news";
+    }
+};
+models.Endorsement.findAll({
+    order: [
+        ['created_at', 'asc']
+    ],
+    include: [
+        models.User,
+        {
+            model: models.Post,
+            include: [
+                {
+                    model: models.Group,
+                    include: [
+                        {
+                            model: models.Community,
+                            where: {
+                                id: communityId
+                            }
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}).then(function (endorsements) {
+    var outFileContent = "";
+    console.log(endorsements.length);
+    outFileContent += "Post endorsements for Community Id: " + communityId + "\n";
+    outFileContent += "User Id, External User Id, State, Group Id, Post Id, Vote Value\n";
+    async.eachSeries(endorsements, function (endorsement, seriesCallback) {
+        var externalUserId = '', state = '';
+        if (endorsement.User.profile_data) {
+            if (endorsement.User.profile_data.trackingParameters && endorsement.User.profile_data.trackingParameters.externalUserId) {
+                externalUserId = endorsement.User.profile_data.trackingParameters.externalUserId;
+            }
+            if (endorsement.User.profile_data.trackingParameters && endorsement.User.profile_data.trackingParameters.state) {
+                state = endorsement.User.profile_data.trackingParameters.state;
+            }
+        }
+        outFileContent += endorsement.User.id + ',' + externalUserId + ',' + state + ',' + endorsement.Post.group_id + ',' + endorsement.post_id + ',' + getType(endorsement.value) + '\n';
+        seriesCallback();
+    }, function (error) {
+        fs.writeFile(outFile, outFileContent, function (err) {
+            if (err) {
+                console.log(err);
+            }
+            console.log("The file was saved!");
+            process.exit();
+        });
+    });
+});
+export {};

package/dist/scripts/exportPointQualitiesForCommunity.js

@@ -0,0 +1,84 @@
+var models = require('../models/index.cjs');
+var async = require('async');
+var _ = require('lodash');
+var fs = require('fs');
+var communityId = process.argv[2];
+var outFile = process.argv[3];
+var clean = function (text) {
+    //console.log("Before: "+ text);
+    if (text) {
+        var newText = text.replace('"', "'").replace('\n', '').replace('\r', '').replace(/(\r\n|\n|\r)/gm, "").replace(/"/gm, "'").replace(',', ';').trim();
+        //console.log("After:" + newText);
+        return newText.replace(/´/g, '');
+    }
+    else {
+        return "empty";
+    }
+};
+var getType = function (value) {
+    if (value > 0) {
+        return "for";
+    }
+    else if (value < 0) {
+        return "against";
+    }
+    else {
+        return "comment/news";
+    }
+};
+models.PointQuality.findAll({
+    order: [
+        ['created_at', 'asc']
+    ],
+    include: [
+        models.User,
+        {
+            model: models.Point,
+            include: [
+                {
+                    model: models.Post,
+                    include: [
+                        {
+                            model: models.Group,
+                            include: [
+                                {
+                                    model: models.Community,
+                                    where: {
+                                        id: communityId
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}).then(function (endorsements) {
+    var outFileContent = "";
+    console.log(endorsements.length);
+    outFileContent += "Point qualities for Community Id: " + communityId + "\n";
+    outFileContent += "User Id, External User Id, State, Group Id, Post Id, Point Id, Vote Value\n";
+    async.eachSeries(endorsements, function (endorsement, seriesCallback) {
+        var externalUserId = '', state = '';
+        if (endorsement.User.profile_data) {
+            if (endorsement.User.profile_data.trackingParameters && endorsement.User.profile_data.trackingParameters.externalUserId) {
+                externalUserId = endorsement.User.profile_data.trackingParameters.externalUserId;
+            }
+            if (endorsement.User.profile_data.trackingParameters && endorsement.User.profile_data.trackingParameters.state) {
+                state = endorsement.User.profile_data.trackingParameters.state;
+            }
+        }
+        outFileContent += endorsement.User.id + ',' + externalUserId + ',' + state + ',' + endorsement.Point.Post.group_id + ',' + endorsement.Point.post_id + ',' + endorsement.Point.id + ',' + getType(endorsement.value) + '\n';
+        seriesCallback();
+    }, function (error) {
+        fs.writeFile(outFile, outFileContent, function (err) {
+            if (err) {
+                console.log(err);
+            }
+            console.log("The file was saved!");
+            process.exit();
+        });
+    });
+});
+export {};