@yrpri/api 9.0.103 → 9.0.104

package/dist/active-citizen/engine/old/exporters/categories_dataset.js

@@ -1,153 +0,0 @@
-var models = require('../../models/index.cjs');
-var async = require('async');
-var fs = require('fs');
-var trainCategoriesCsv;
-var testCategoriesCsv;
-var classesCategoriesCsv;
-var trainCategoriesCsvFilename = 'datasets/better_reykjavik/categories/train.csv';
-var testCategoriesCsvFilename = 'datasets/better_reykjavik/categories/test.csv';
-var classesCategoriesCsvFilename = 'datasets/better_reykjavik/categories/classes.csv';
-var categories = {};
-var categoriesIds = [];
-MAX_CATEGORY_LENGTH = 1700;
-var clean = require('./dataset_tools').clean;
-var shuffleArray = require('./dataset_tools').shuffleArray;
-var replaceBetterReykjavikCategoryId = require('./dataset_tools').replaceBetterReykjavikCategoryId;
-async.series([
-    function (callback) {
-        var categoriesCsvRows = [];
-        models.Post.findAll({
-            where: {
-                status: 'published'
-            },
-            include: [
-                {
-                    model: models.Point,
-                    where: {
-                        status: 'published'
-                    }
-                },
-                {
-                    model: models.Group,
-                    required: true,
-                    include: [
-                        {
-                            model: models.Community,
-                            required: true,
-                            include: [
-                                {
-                                    model: models.Domain,
-                                    required: true,
-                                    where: {
-                                        id: 1
-                                    }
-                                }
-                            ]
-                        }
-                    ]
-                }
-            ]
-        }).then(function (posts) {
-            console.log('Found ' + posts.length + " posts");
-            async.eachSeries(posts, function (post, seriesCallback) {
-                var newId = replaceBetterReykjavikCategoryId(post.category_id);
-                if (newId && newId != 11) {
-                    if (!categories[newId]) {
-                        categories[newId] = [];
-                        categoriesIds.push(newId);
-                    }
-                    var content;
-                    if (post.description) {
-                        content = '"' + clean(post.name) + ' ' + clean(post.description) + '"';
-                    }
-                    else {
-                        content = '"' + clean(post.name) + '"';
-                    }
-                    if (content.indexOf('Lorem Ipsum har') == -1) {
-                        categories[newId].push(content);
-                    }
-                    async.eachSeries(post.Points, function (point, innerSeriesCallback) {
-                        console.log(point.status);
-                        if (point.value != 0) {
-                            content = '"' + clean(point.content) + '"';
-                            if (content != "" && content.length > 17) {
-                                if (content.indexOf('Mypoint my point') == -1 &&
-                                    content.indexOf('Point against Point') == -1) {
-                                    categories[newId].push(content);
-                                }
-                            }
-                        }
-                        innerSeriesCallback();
-                    }, function () {
-                        seriesCallback();
-                    });
-                }
-                else {
-                    seriesCallback();
-                }
-            }, function () {
-                async.eachSeries(categoriesIds, function (category_id, seriesCallback) {
-                    console.log(category_id);
-                    console.log("-----" + category_id + "---------------------------: " + categories[category_id].length);
-                    if (categories[category_id].length > MAX_CATEGORY_LENGTH) {
-                        categories[category_id] = categories[category_id].splice(0, MAX_CATEGORY_LENGTH);
-                        console.log("-----" + category_id + "---------------------------: " + categories[category_id].length);
-                    }
-                    async.eachSeries(categories[category_id], function (post, innerSeriesCallback) {
-                        categoriesCsvRows.push(category_id + ',' + post);
-                        console.log("Key: " + category_id + " value: " + post);
-                        innerSeriesCallback();
-                    }, function () {
-                        seriesCallback();
-                    });
-                }, function () {
-                    trainCategoriesCsv = shuffleArray(categoriesCsvRows);
-                    testCategoriesCsv = trainCategoriesCsv.splice(0, categoriesCsvRows.length * 0.1);
-                    callback();
-                });
-            });
-        }).catch(function (error) {
-            console.log("ERROR: " + error);
-        });
-    },
-    function (callback) {
-        fs.writeFile(trainCategoriesCsvFilename, trainCategoriesCsv.join('\n'), function (err) {
-            if (err) {
-                console.log(err);
-            }
-            callback();
-        });
-    },
-    function (callback) {
-        fs.writeFile(testCategoriesCsvFilename, testCategoriesCsv.join('\n'), function (err) {
-            if (err) {
-                console.log(err);
-            }
-            callback();
-        });
-    },
-    function (callback) {
-        classesCategoriesCsv = [];
-        async.eachSeries(categoriesIds, function (category_id, seriesCallback) {
-            models.Category.findOne({
-                where: { id: category_id }
-            }).then(function (category) {
-                classesCategoriesCsv.push(category.id + ',' + category.name);
-                seriesCallback();
-            });
-        }, function () {
-            callback();
-        });
-    },
-    function (callback) {
-        fs.writeFile(classesCategoriesCsvFilename, classesCategoriesCsv.join('\n'), function (err) {
-            if (err) {
-                console.log(err);
-            }
-            callback();
-        });
-    }
-], function (error) {
-    console.log("FINISHED :)");
-});
-export {};

package/dist/active-citizen/engine/old/exporters/dataset_tools.js

@@ -1,80 +0,0 @@
-var removeDiacritics = require('diacritics').remove;
-var shuffleArray = function (array) {
-    for (var i = array.length - 1; i > 0; i--) {
-        var j = Math.floor(Math.random() * (i + 1));
-        var temp = array[i];
-        array[i] = array[j];
-        array[j] = temp;
-    }
-    return array;
-};
-var clean = function (string) {
-    string = string.replace(/(\r\n|\n|\r)/gm, " ");
-    string = string.replace(',', ' ');
-    string = string.replace('\n', ' ');
-    string = string.replace(/\'/g, ' ');
-    string = string.replace(/\,/g, ' ');
-    string = string.replace(/\(/g, ' ');
-    string = string.replace(/\=/g, ' ');
-    string = string.replace(/\)/g, ' ');
-    string = string.replace(/\-/g, ' ');
-    string = string.replace(/\./g, ' ');
-    string = string.replace(/['"]+/g, '');
-    string = string.replace(/<a\b[^>]*>(.*?)<\/a>/i, " ");
-    string = removeDiacritics(string);
-    string = string.replace(/[^A-Za-z0-9(),!?\'\`]/, ' ');
-    string = string.replace(/[^\x00-\x7F]/g, " ");
-    string = string.replace(/\s+/g, ' ').trim();
-    return string;
-};
-var replaceBetterReykjavikCategoryId = function (id) {
-    if ([1, 2, 19, 24, 18, 16, 20, 23, 17, 21, 13, 25, 22, 14].indexOf(id) > -1) {
-        return 2;
-    }
-    else {
-        switch (id) {
-            case 4:
-                return 1;
-                break;
-            case 3:
-                return 3;
-                break;
-            case 15:
-                return 4;
-                break;
-            case 11:
-                return 5;
-                break;
-            case 8:
-                return 6;
-                break;
-            case 9:
-                return 7;
-                break;
-            case 10:
-                return 8;
-                break;
-            case 12:
-                return 9;
-                break;
-            case 5:
-                return 10;
-                break;
-            case 26:
-                return 11;
-                break;
-            case 6:
-                return 12;
-                break;
-            case 7:
-                return 13;
-                break;
-        }
-    }
-};
-module.exports = {
-    replaceBetterReykjavikCategoryId: replaceBetterReykjavikCategoryId,
-    clean: clean,
-    shuffleArray: shuffleArray
-};
-export {};

package/dist/active-citizen/engine/old/exporters/sentiment_dataset.js

@@ -1,157 +0,0 @@
-var models = require('../../models/index.cjs');
-var async = require('async');
-var fs = require('fs');
-var trainCategoriesCsv;
-var testCategoriesCsv;
-var classesCategoriesCsv;
-var trainCsvFilename = 'datasets/better_reykjavik/sentiment/train.csv';
-var testTrainCsvFilename = 'datasets/better_reykjavik/sentiment/test.csv';
-var classesTrainCsvFilename = 'datasets/better_reykjavik/sentiment/classes.csv';
-var classesTrainPositiveCsvFilename = 'datasets/better_reykjavik/sentiment/positive.polarity';
-var classesTrainNegativeCsvFilename = 'datasets/better_reykjavik/sentiment/negative.polarity';
-var positive = [];
-var negative = [];
-var categories = { 0: [], 1: [] };
-var categoriesIds = [0, 1];
-MAX_SENTIMENT_LENGTH = 700;
-var clean = require('./dataset_tools').clean;
-var shuffleArray = require('./dataset_tools').shuffleArray;
-async.series([
-    function (callback) {
-        var categoriesCsvRows = [];
-        models.Post.findAll({
-            where: {
-                status: 'published'
-            },
-            include: [
-                {
-                    model: models.Point,
-                    where: {
-                        status: 'published'
-                    }
-                },
-                {
-                    model: models.Group,
-                    required: true,
-                    where: {
-                        access: models.Group.ACCESS_PUBLIC
-                    },
-                    include: [
-                        {
-                            model: models.Community,
-                            required: true,
-                            where: {
-                                access: models.Community.ACCESS_PUBLIC
-                            },
-                            include: [
-                                {
-                                    model: models.Domain,
-                                    required: true,
-                                    where: {
-                                        id: { $in: [1, 2] }
-                                    }
-                                }
-                            ]
-                        }
-                    ]
-                }
-            ]
-        }).then(function (posts) {
-            console.log('Found ' + posts.length + " posts");
-            async.eachSeries(posts, function (post, seriesCallback) {
-                async.eachSeries(post.Points, function (point, innerSeriesCallback) {
-                    if (point.value != 0) {
-                        content = '"' + clean(point.content) + '"';
-                        if (content != "" && content.length > 17) {
-                            if (content.indexOf('Mypoint my point') == -1 &&
-                                content.indexOf('Point against Point') == -1) {
-                                if (point.value > 0) {
-                                    categories['0'].push(content);
-                                    positive.push(clean(point.content));
-                                }
-                                else if (point.value < 0) {
-                                    categories['1'].push(content);
-                                    negative.push(clean(point.content));
-                                }
-                            }
-                        }
-                    }
-                    innerSeriesCallback();
-                }, function () {
-                    seriesCallback();
-                });
-            }, function () {
-                async.eachSeries(categoriesIds, function (category_id, seriesCallback) {
-                    console.log(category_id);
-                    console.log("-----" + category_id + "---------------------------: " + categories[category_id].length);
-                    if (categories[category_id].length > MAX_SENTIMENT_LENGTH) {
-                        categories[category_id] = categories[category_id].splice(0, MAX_SENTIMENT_LENGTH);
-                        console.log("-----" + category_id + "---------------------------: " + categories[category_id].length);
-                    }
-                    async.eachSeries(categories[category_id], function (post, innerSeriesCallback) {
-                        categoriesCsvRows.push(category_id + ',' + post);
-                        console.log("Key: " + category_id + " value: " + post);
-                        innerSeriesCallback();
-                    }, function () {
-                        seriesCallback();
-                    });
-                }, function () {
-                    trainCategoriesCsv = shuffleArray(categoriesCsvRows);
-                    testCategoriesCsv = trainCategoriesCsv.splice(0, categoriesCsvRows.length * 0.1);
-                    callback();
-                });
-            });
-        }).catch(function (error) {
-            console.log("ERROR: " + error);
-        });
-    },
-    function (callback) {
-        fs.writeFile(trainCsvFilename, trainCategoriesCsv.join('\n'), function (err) {
-            if (err) {
-                console.log(err);
-            }
-            callback();
-        });
-    },
-    function (callback) {
-        fs.writeFile(testTrainCsvFilename, testCategoriesCsv.join('\n'), function (err) {
-            if (err) {
-                console.log(err);
-            }
-            callback();
-        });
-    },
-    function (callback) {
-        negative = positive.splice(0, MAX_SENTIMENT_LENGTH);
-        fs.writeFile(classesTrainPositiveCsvFilename, positive.join('\n'), function (err) {
-            if (err) {
-                console.log(err);
-            }
-            callback();
-        });
-    },
-    function (callback) {
-        negative = negative.splice(0, MAX_SENTIMENT_LENGTH);
-        fs.writeFile(classesTrainNegativeCsvFilename, negative.join('\n'), function (err) {
-            if (err) {
-                console.log(err);
-            }
-            callback();
-        });
-    },
-    function (callback) {
-        classesCategoriesCsv = ["0,Positive", "1,Negative"];
-        callback();
-    },
-    function (callback) {
-        fs.writeFile(classesTrainCsvFilename, classesCategoriesCsv.join('\n'), function (err) {
-            if (err) {
-                console.log(err);
-            }
-            callback();
-        });
-    }
-], function (error) {
-    console.log("FINISHED :)");
-});
-export {};

package/dist/active-citizen/engine/recommendations/events_importer.cjs

@@ -1,139 +0,0 @@
-"use strict";
-const models = require('../../../models/index.cjs');
-const _ = require('lodash');
-const async = require('async');
-const log = require('../../../utils/logger.cjs');
-const createAction = require('./events_manager.cjs').createAction;
-const createManyActions = require('./events_manager.cjs').createManyActions;
-let postUpdateAsyncLimit = 42;
-var lineCrCounter = 0;
-var processDots = function () {
-    if (lineCrCounter > 250) {
-        process.stdout.write("\n");
-        lineCrCounter = 1;
-    }
-    else {
-        process.stdout.write(".");
-        lineCrCounter += 1;
-    }
-};
-const chunkArray = (myArray, chunk_size) => {
-    var index = 0;
-    var arrayLength = myArray.length;
-    var tempArray = [];
-    for (index = 0; index < arrayLength; index += chunk_size) {
-        const myChunk = myArray.slice(index, index + chunk_size);
-        // Do something if you want with the group
-        tempArray.push(myChunk);
-    }
-    return tempArray;
-};
-const importAllActionsFor = function (model, where, include, action, done, attributes) {
-    log.info('AcImportAllActionsFor', { action: action, model: model, where: where, include: include });
-    model.findAll({
-        where: where,
-        include: include,
-        attributes: attributes
-    }).then(function (objects) {
-        lineCrCounter = 0;
-        const chunks = chunkArray(objects, 1000);
-        async.eachOfLimit(chunks, 1, (objects, index, callback) => {
-            const actionsToSend = [];
-            async.eachOfSeries(objects, (object, index, seriesCallback) => {
-                let postId;
-                if (action.indexOf('help') > -1) {
-                    postId = object.Point.Post.id;
-                }
-                else if (action.indexOf('post') > -1) {
-                    postId = object.id;
-                }
-                else {
-                    postId = object.Post.id;
-                }
-                if (postId) {
-                    const esId = `${postId}-${object.user_id}-${action}`;
-                    actionsToSend.push({
-                        postId,
-                        userId: object.user_id,
-                        date: object.created_at.toISOString(),
-                        user_agent: object.user_agent,
-                        ip_address: object.ip_address,
-                        action,
-                        esId
-                    });
-                    processDots();
-                    seriesCallback();
-                }
-                else {
-                    console.error("Can't find id for object: " + object);
-                    seriesCallback();
-                }
-            }, error => {
-                if (!error) {
-                    createManyActions(JSON.parse(JSON.stringify(actionsToSend)), callback);
-                }
-                else {
-                    callback(error);
-                }
-            });
-        }, function (error) {
-            console.log(error);
-            console.log("\n FIN");
-            done();
-        });
-    });
-};
-var importAll = function (done) {
-    async.series([
-        function (callback) {
-            importAllActionsFor(models.Post, {}, [], 'new-post', function () {
-                callback();
-            }, ['id', 'user_id', 'created_at', 'ip_address', 'user_agent']);
-        },
-        function (callback) {
-            importAllActionsFor(models.Endorsement, { value: { $gt: 0 } }, [{ model: models.Post, attributes: ['id'] }], 'endorse', function () {
-                callback();
-            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
-        },
-        function (callback) {
-            importAllActionsFor(models.Endorsement, { value: { $lt: 0 } }, [{ model: models.Post, attributes: ['id'] }], 'oppose', function () {
-                callback();
-            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
-        },
-        function (callback) {
-            importAllActionsFor(models.Point, { value: { $ne: 0 } }, [{ model: models.Post, attributes: ['id'] }], 'new-point', function () {
-                callback();
-            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
-        },
-        function (callback) {
-            importAllActionsFor(models.Point, { value: 0 }, [{ model: models.Post, attributes: ['id'] }], 'new-point-comment', function () {
-                callback();
-            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
-        },
-        function (callback) {
-            importAllActionsFor(models.PointQuality, { value: { $gt: 0 } }, [{
-                    model: models.Point,
-                    attributes: ['id', 'value'],
-                    include: [{ model: models.Post, attributes: ['id'] }]
-                }], 'point-helpful', function () {
-                callback();
-            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
-        },
-        function (callback) {
-            importAllActionsFor(models.PointQuality, { value: { $lt: 0 } }, [{
-                    model: models.Point,
-                    attributes: ['id', 'value'],
-                    include: [models.Post]
-                }], 'point-unhelpful', function () {
-                callback();
-            }, ['id', 'user_id', 'created_at', 'value', 'ip_address', 'user_agent']);
-        }
-    ], function () {
-        console.log("FIN");
-        done();
-    });
-};
-importAll(function () {
-    console.log("Done importing all");
-    process.exit();
-});