@yrpri/api 9.0.101 → 9.0.103

package/dist/active-citizen/engine/old/exporters/categories_dataset.js

@@ -0,0 +1,153 @@
+var models = require('../../models/index.cjs');
+var async = require('async');
+var fs = require('fs');
+var trainCategoriesCsv;
+var testCategoriesCsv;
+var classesCategoriesCsv;
+var trainCategoriesCsvFilename = 'datasets/better_reykjavik/categories/train.csv';
+var testCategoriesCsvFilename = 'datasets/better_reykjavik/categories/test.csv';
+var classesCategoriesCsvFilename = 'datasets/better_reykjavik/categories/classes.csv';
+var categories = {};
+var categoriesIds = [];
+MAX_CATEGORY_LENGTH = 1700;
+var clean = require('./dataset_tools').clean;
+var shuffleArray = require('./dataset_tools').shuffleArray;
+var replaceBetterReykjavikCategoryId = require('./dataset_tools').replaceBetterReykjavikCategoryId;
+async.series([
+    function (callback) {
+        var categoriesCsvRows = [];
+        models.Post.findAll({
+            where: {
+                status: 'published'
+            },
+            include: [
+                {
+                    model: models.Point,
+                    where: {
+                        status: 'published'
+                    }
+                },
+                {
+                    model: models.Group,
+                    required: true,
+                    include: [
+                        {
+                            model: models.Community,
+                            required: true,
+                            include: [
+                                {
+                                    model: models.Domain,
+                                    required: true,
+                                    where: {
+                                        id: 1
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                }
+            ]
+        }).then(function (posts) {
+            console.log('Found ' + posts.length + " posts");
+            async.eachSeries(posts, function (post, seriesCallback) {
+                var newId = replaceBetterReykjavikCategoryId(post.category_id);
+                if (newId && newId != 11) {
+                    if (!categories[newId]) {
+                        categories[newId] = [];
+                        categoriesIds.push(newId);
+                    }
+                    var content;
+                    if (post.description) {
+                        content = '"' + clean(post.name) + ' ' + clean(post.description) + '"';
+                    }
+                    else {
+                        content = '"' + clean(post.name) + '"';
+                    }
+                    if (content.indexOf('Lorem Ipsum har') == -1) {
+                        categories[newId].push(content);
+                    }
+                    async.eachSeries(post.Points, function (point, innerSeriesCallback) {
+                        console.log(point.status);
+                        if (point.value != 0) {
+                            content = '"' + clean(point.content) + '"';
+                            if (content != "" && content.length > 17) {
+                                if (content.indexOf('Mypoint my point') == -1 &&
+                                    content.indexOf('Point against Point') == -1) {
+                                    categories[newId].push(content);
+                                }
+                            }
+                        }
+                        innerSeriesCallback();
+                    }, function () {
+                        seriesCallback();
+                    });
+                }
+                else {
+                    seriesCallback();
+                }
+            }, function () {
+                async.eachSeries(categoriesIds, function (category_id, seriesCallback) {
+                    console.log(category_id);
+                    console.log("-----" + category_id + "---------------------------: " + categories[category_id].length);
+                    if (categories[category_id].length > MAX_CATEGORY_LENGTH) {
+                        categories[category_id] = categories[category_id].splice(0, MAX_CATEGORY_LENGTH);
+                        console.log("-----" + category_id + "---------------------------: " + categories[category_id].length);
+                    }
+                    async.eachSeries(categories[category_id], function (post, innerSeriesCallback) {
+                        categoriesCsvRows.push(category_id + ',' + post);
+                        console.log("Key: " + category_id + " value: " + post);
+                        innerSeriesCallback();
+                    }, function () {
+                        seriesCallback();
+                    });
+                }, function () {
+                    trainCategoriesCsv = shuffleArray(categoriesCsvRows);
+                    testCategoriesCsv = trainCategoriesCsv.splice(0, categoriesCsvRows.length * 0.1);
+                    callback();
+                });
+            });
+        }).catch(function (error) {
+            console.log("ERROR: " + error);
+        });
+    },
+    function (callback) {
+        fs.writeFile(trainCategoriesCsvFilename, trainCategoriesCsv.join('\n'), function (err) {
+            if (err) {
+                console.log(err);
+            }
+            callback();
+        });
+    },
+    function (callback) {
+        fs.writeFile(testCategoriesCsvFilename, testCategoriesCsv.join('\n'), function (err) {
+            if (err) {
+                console.log(err);
+            }
+            callback();
+        });
+    },
+    function (callback) {
+        classesCategoriesCsv = [];
+        async.eachSeries(categoriesIds, function (category_id, seriesCallback) {
+            models.Category.findOne({
+                where: { id: category_id }
+            }).then(function (category) {
+                classesCategoriesCsv.push(category.id + ',' + category.name);
+                seriesCallback();
+            });
+        }, function () {
+            callback();
+        });
+    },
+    function (callback) {
+        fs.writeFile(classesCategoriesCsvFilename, classesCategoriesCsv.join('\n'), function (err) {
+            if (err) {
+                console.log(err);
+            }
+            callback();
+        });
+    }
+], function (error) {
+    console.log("FINISHED :)");
+});
+export {};

package/dist/active-citizen/engine/old/exporters/dataset_tools.js

@@ -0,0 +1,80 @@
+var removeDiacritics = require('diacritics').remove;
+var shuffleArray = function (array) {
+    for (var i = array.length - 1; i > 0; i--) {
+        var j = Math.floor(Math.random() * (i + 1));
+        var temp = array[i];
+        array[i] = array[j];
+        array[j] = temp;
+    }
+    return array;
+};
+var clean = function (string) {
+    string = string.replace(/(\r\n|\n|\r)/gm, " ");
+    string = string.replace(',', ' ');
+    string = string.replace('\n', ' ');
+    string = string.replace(/\'/g, ' ');
+    string = string.replace(/\,/g, ' ');
+    string = string.replace(/\(/g, ' ');
+    string = string.replace(/\=/g, ' ');
+    string = string.replace(/\)/g, ' ');
+    string = string.replace(/\-/g, ' ');
+    string = string.replace(/\./g, ' ');
+    string = string.replace(/['"]+/g, '');
+    string = string.replace(/<a\b[^>]*>(.*?)<\/a>/i, " ");
+    string = removeDiacritics(string);
+    string = string.replace(/[^A-Za-z0-9(),!?\'\`]/, ' ');
+    string = string.replace(/[^\x00-\x7F]/g, " ");
+    string = string.replace(/\s+/g, ' ').trim();
+    return string;
+};
+var replaceBetterReykjavikCategoryId = function (id) {
+    if ([1, 2, 19, 24, 18, 16, 20, 23, 17, 21, 13, 25, 22, 14].indexOf(id) > -1) {
+        return 2;
+    }
+    else {
+        switch (id) {
+            case 4:
+                return 1;
+                break;
+            case 3:
+                return 3;
+                break;
+            case 15:
+                return 4;
+                break;
+            case 11:
+                return 5;
+                break;
+            case 8:
+                return 6;
+                break;
+            case 9:
+                return 7;
+                break;
+            case 10:
+                return 8;
+                break;
+            case 12:
+                return 9;
+                break;
+            case 5:
+                return 10;
+                break;
+            case 26:
+                return 11;
+                break;
+            case 6:
+                return 12;
+                break;
+            case 7:
+                return 13;
+                break;
+        }
+    }
+};
+module.exports = {
+    replaceBetterReykjavikCategoryId: replaceBetterReykjavikCategoryId,
+    clean: clean,
+    shuffleArray: shuffleArray
+};
+export {};

package/dist/active-citizen/engine/old/exporters/sentiment_dataset.js

@@ -0,0 +1,157 @@
+var models = require('../../models/index.cjs');
+var async = require('async');
+var fs = require('fs');
+var trainCategoriesCsv;
+var testCategoriesCsv;
+var classesCategoriesCsv;
+var trainCsvFilename = 'datasets/better_reykjavik/sentiment/train.csv';
+var testTrainCsvFilename = 'datasets/better_reykjavik/sentiment/test.csv';
+var classesTrainCsvFilename = 'datasets/better_reykjavik/sentiment/classes.csv';
+var classesTrainPositiveCsvFilename = 'datasets/better_reykjavik/sentiment/positive.polarity';
+var classesTrainNegativeCsvFilename = 'datasets/better_reykjavik/sentiment/negative.polarity';
+var positive = [];
+var negative = [];
+var categories = { 0: [], 1: [] };
+var categoriesIds = [0, 1];
+MAX_SENTIMENT_LENGTH = 700;
+var clean = require('./dataset_tools').clean;
+var shuffleArray = require('./dataset_tools').shuffleArray;
+async.series([
+    function (callback) {
+        var categoriesCsvRows = [];
+        models.Post.findAll({
+            where: {
+                status: 'published'
+            },
+            include: [
+                {
+                    model: models.Point,
+                    where: {
+                        status: 'published'
+                    }
+                },
+                {
+                    model: models.Group,
+                    required: true,
+                    where: {
+                        access: models.Group.ACCESS_PUBLIC
+                    },
+                    include: [
+                        {
+                            model: models.Community,
+                            required: true,
+                            where: {
+                                access: models.Community.ACCESS_PUBLIC
+                            },
+                            include: [
+                                {
+                                    model: models.Domain,
+                                    required: true,
+                                    where: {
+                                        id: { $in: [1, 2] }
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                }
+            ]
+        }).then(function (posts) {
+            console.log('Found ' + posts.length + " posts");
+            async.eachSeries(posts, function (post, seriesCallback) {
+                async.eachSeries(post.Points, function (point, innerSeriesCallback) {
+                    if (point.value != 0) {
+                        content = '"' + clean(point.content) + '"';
+                        if (content != "" && content.length > 17) {
+                            if (content.indexOf('Mypoint my point') == -1 &&
+                                content.indexOf('Point against Point') == -1) {
+                                if (point.value > 0) {
+                                    categories['0'].push(content);
+                                    positive.push(clean(point.content));
+                                }
+                                else if (point.value < 0) {
+                                    categories['1'].push(content);
+                                    negative.push(clean(point.content));
+                                }
+                            }
+                        }
+                    }
+                    innerSeriesCallback();
+                }, function () {
+                    seriesCallback();
+                });
+            }, function () {
+                async.eachSeries(categoriesIds, function (category_id, seriesCallback) {
+                    console.log(category_id);
+                    console.log("-----" + category_id + "---------------------------: " + categories[category_id].length);
+                    if (categories[category_id].length > MAX_SENTIMENT_LENGTH) {
+                        categories[category_id] = categories[category_id].splice(0, MAX_SENTIMENT_LENGTH);
+                        console.log("-----" + category_id + "---------------------------: " + categories[category_id].length);
+                    }
+                    async.eachSeries(categories[category_id], function (post, innerSeriesCallback) {
+                        categoriesCsvRows.push(category_id + ',' + post);
+                        console.log("Key: " + category_id + " value: " + post);
+                        innerSeriesCallback();
+                    }, function () {
+                        seriesCallback();
+                    });
+                }, function () {
+                    trainCategoriesCsv = shuffleArray(categoriesCsvRows);
+                    testCategoriesCsv = trainCategoriesCsv.splice(0, categoriesCsvRows.length * 0.1);
+                    callback();
+                });
+            });
+        }).catch(function (error) {
+            console.log("ERROR: " + error);
+        });
+    },
+    function (callback) {
+        fs.writeFile(trainCsvFilename, trainCategoriesCsv.join('\n'), function (err) {
+            if (err) {
+                console.log(err);
+            }
+            callback();
+        });
+    },
+    function (callback) {
+        fs.writeFile(testTrainCsvFilename, testCategoriesCsv.join('\n'), function (err) {
+            if (err) {
+                console.log(err);
+            }
+            callback();
+        });
+    },
+    function (callback) {
+        negative = positive.splice(0, MAX_SENTIMENT_LENGTH);
+        fs.writeFile(classesTrainPositiveCsvFilename, positive.join('\n'), function (err) {
+            if (err) {
+                console.log(err);
+            }
+            callback();
+        });
+    },
+    function (callback) {
+        negative = negative.splice(0, MAX_SENTIMENT_LENGTH);
+        fs.writeFile(classesTrainNegativeCsvFilename, negative.join('\n'), function (err) {
+            if (err) {
+                console.log(err);
+            }
+            callback();
+        });
+    },
+    function (callback) {
+        classesCategoriesCsv = ["0,Positive", "1,Negative"];
+        callback();
+    },
+    function (callback) {
+        fs.writeFile(classesTrainCsvFilename, classesCategoriesCsv.join('\n'), function (err) {
+            if (err) {
+                console.log(err);
+            }
+            callback();
+        });
+    }
+], function (error) {
+    console.log("FINISHED :)");
+});
+export {};

package/dist/active-citizen/engine/recommendations/events_importer.cjs

@@ -0,0 +1,139 @@
+"use strict";
+const models = require('../../../models/index.cjs');
+const _ = require('lodash');
+const async = require('async');
+const log = require('../../../utils/logger.cjs');
+const createAction = require('./events_manager.cjs').createAction;
+const createManyActions = require('./events_manager.cjs').createManyActions;
+let postUpdateAsyncLimit = 42;
+var lineCrCounter = 0;
+var processDots = function () {
+    if (lineCrCounter > 250) {
+        process.stdout.write("\n");
+        lineCrCounter = 1;
+    }
+    else {
+        process.stdout.write(".");
+        lineCrCounter += 1;
+    }
+};
+const chunkArray = (myArray, chunk_size) => {
+    var index = 0;
+    var arrayLength = myArray.length;
+    var tempArray = [];
+    for (index = 0; index < arrayLength; index += chunk_size) {
+        const myChunk = myArray.slice(index, index + chunk_size);
+        // Do something if you want with the group
+        tempArray.push(myChunk);
+    }
+    return tempArray;
+};
+const importAllActionsFor = function (model, where, include, action, done, attributes) {
+    log.info('AcImportAllActionsFor', { action: action, model: model, where: where, include: include });
+    model.findAll({
+        where: where,
+        include: include,
+        attributes: attributes
+    }).then(function (objects) {
+        lineCrCounter = 0;
+        const chunks = chunkArray(objects, 1000);
+        async.eachOfLimit(chunks, 1, (objects, index, callback) => {
+            const actionsToSend = [];
+            async.eachOfSeries(objects, (object, index, seriesCallback) => {
+                let postId;
+                if (action.indexOf('help') > -1) {
+                    postId = object.Point.Post.id;
+                }
+                else if (action.indexOf('post') > -1) {
+                    postId = object.id;
+                }
+                else {
+                    postId = object.Post.id;
+                }
+                if (postId) {
+                    const esId = `${postId}-${object.user_id}-${action}`;
+                    actionsToSend.push({
+                        postId,
+                        userId: object.user_id,
+                        date: object.created_at.toISOString(),
+                        user_agent: object.user_agent,
+                        ip_address: object.ip_address,
+                        action,
+                        esId
+                    });
+                    processDots();
+                    seriesCallback();
+                }
+                else {
+                    console.error("Can't find id for object: " + object);
+                    seriesCallback();
+                }
+            }, error => {
+                if (!error) {
+                    createManyActions(JSON.parse(JSON.stringify(actionsToSend)), callback);
+                }
+                else {
+                    callback(error);
+                }
+            });
+        }, function (error) {
+            console.log(error);
+            console.log("\n FIN");
+            done();
+        });
+    });
+};
+var importAll = function (done) {
+    async.series([
+        function (callback) {
+            importAllActionsFor(models.Post, {}, [], 'new-post', function () {
+                callback();
+            }, ['id', 'user_id', 'created_at', 'ip_address', 'user_agent']);
+        },
+        function (callback) {
+            importAllActionsFor(models.Endorsement, { value: { $gt: 0 } }, [{ model: models.Post, attributes: ['id'] }], 'endorse', function () {
+                callback();
+            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
+        },
+        function (callback) {
+            importAllActionsFor(models.Endorsement, { value: { $lt: 0 } }, [{ model: models.Post, attributes: ['id'] }], 'oppose', function () {
+                callback();
+            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
+        },
+        function (callback) {
+            importAllActionsFor(models.Point, { value: { $ne: 0 } }, [{ model: models.Post, attributes: ['id'] }], 'new-point', function () {
+                callback();
+            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
+        },
+        function (callback) {
+            importAllActionsFor(models.Point, { value: 0 }, [{ model: models.Post, attributes: ['id'] }], 'new-point-comment', function () {
+                callback();
+            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
+        },
+        function (callback) {
+            importAllActionsFor(models.PointQuality, { value: { $gt: 0 } }, [{
+                    model: models.Point,
+                    attributes: ['id', 'value'],
+                    include: [{ model: models.Post, attributes: ['id'] }]
+                }], 'point-helpful', function () {
+                callback();
+            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
+        },
+        function (callback) {
+            importAllActionsFor(models.PointQuality, { value: { $lt: 0 } }, [{
+                    model: models.Point,
+                    attributes: ['id', 'value'],
+                    include: [models.Post]
+                }], 'point-unhelpful', function () {
+                callback();
+            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
+        }
+    ], function () {
+        console.log("FIN");
+        done();
+    });
+};
+importAll(function () {
+    console.log("Done importing all");
+    process.exit();
+});