@yrpri/api 9.0.103 → 9.0.104

package/dist/active-citizen/engine/analytics/export_anon_community_activities.js

@@ -1,334 +0,0 @@
-const models = require('../../../models/index.cjs');
-const _ = require('lodash');
-const async = require('async');
-const log = require('../../../utils/logger.cjs');
-const crypto = require('crypto');
-const fs = require('fs');
-//const request = require('request');
-const request = {
-    post: (data, done) => {
-        console.log(JSON.stringify(data, null, 4));
-        done();
-    }
-};
-const clean = (text) => {
-    //console.log("Before: "+ text);
-    var newText = text.replace('"', "'").replace('\n', '').replace('\r', '').replace(/(\r\n|\n|\r)/gm, "").replace(/"/gm, "'").replace(/,/, ';').trim();
-    //console.log("After:" + newText);
-    return newText.replace(/´/g, '');
-};
-const getEncryptedId = (id, key, iv) => {
-    let cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(key), iv);
-    let encrypted = cipher.update(id.toString());
-    encrypted = Buffer.concat([encrypted, cipher.final()]);
-    return encrypted.toString('hex');
-};
-let csvPostsTxt = "Id,Date,Name,NameEn,Description,DescriptionEn,ToxicityScore\n";
-const saveAnonymousPost = (post, accessKey, done) => {
-    let expiryDate = new Date();
-    expiryDate.setDate(expiryDate.getDate() + 7);
-    const properties = {
-        id: post.id,
-        name: post.name,
-        name_en: post.name_en,
-        date: post.created_at.toISOString(),
-        description: post.description,
-        description_en: post.description_en,
-        toxicity_score: post.toxicity_score,
-        access_key: accessKey,
-        access_expires_at: expiryDate.toISOString()
-    };
-    csvPostsTxt += post.id + ',' + properties.date + ',"' + clean(post.name) + '","' + clean(post.name_en) + '",' + '"' + clean(post.description) + '",' + '"' + clean(post.description_en) + '",' + post.toxicity_score + '\n';
-    const options = {
-        url: process.env["AC_ANALYTICS_BASE_URL"] + "anonPosts/" + process.env.AC_ANALYTICS_CLUSTER_ID + "/" + `${accessKey}${post.id}`,
-        headers: {
-            'X-API-KEY': process.env["AC_ANALYTICS_KEY"]
-        },
-        json: properties
-    };
-    request.post(options, (error) => {
-        done(error);
-    });
-};
-let csvPointsTxt = "Id,Date,Content,ContentEn,Value,ToxicityScore\n";
-const saveAnonymousPoint = (point, accessKey, done) => {
-    let expiryDate = new Date();
-    expiryDate.setDate(expiryDate.getDate() + 7);
-    const properties = {
-        id: point.id,
-        date: point.created_at.toISOString(),
-        content: point.content,
-        content_en: point.content_en,
-        value: point.value,
-        access_key: accessKey,
-        toxicity_score: point.toxicity_score,
-        access_expires_at: expiryDate.toISOString()
-    };
-    csvPointsTxt += point.id + ',' + properties.date + ',"' + clean(point.content) + '",' + '"' + clean(point.content_en) + '",' + point.value + ',' + point.toxicity_score + '\n';
-    const options = {
-        url: process.env["AC_ANALYTICS_BASE_URL"] + "anonPoints/" + process.env.AC_ANALYTICS_CLUSTER_ID + "/" + `${accessKey}${point.id}`,
-        headers: {
-            'X-API-KEY': process.env["AC_ANALYTICS_KEY"]
-        },
-        json: properties
-    };
-    request.post(options, (error) => {
-        done(error);
-    });
-};
-let csvActivitiesTxt = "Id,Date,Type,PointId,PostId,UserId\n";
-const saveActivity = (activity, accessKey, done) => {
-    let expiryDate = new Date();
-    expiryDate.setDate(expiryDate.getDate() + 7);
-    const properties = {
-        id: activity.id,
-        date: activity.created_at.toISOString(),
-        type: activity.type,
-        point_id: activity.point_id,
-        post_id: activity.post_id,
-        user_id: activity.user_id,
-        access_key: accessKey,
-        access_expires_at: expiryDate.toISOString()
-    };
-    csvActivitiesTxt += activity.id + ',' + properties.date + ',"' + activity.type + '",' + '"' + activity.point_id + '",' + '"' + activity.post_id + '",' + '"' + activity.user_id + '"\n';
-    const options = {
-        url: process.env["AC_ANALYTICS_BASE_URL"] + "anonActivities/" + process.env.AC_ANALYTICS_CLUSTER_ID + "/" + `${accessKey}${activity.id}`,
-        headers: {
-            'X-API-KEY': process.env["AC_ANALYTICS_KEY"]
-        },
-        json: properties
-    };
-    request.post(options, (error) => {
-        done(error);
-    });
-};
-let csvUsersTxt = "Id\n";
-const saveAnonymousUser = (user, accessKey, done) => {
-    let expiryDate = new Date();
-    expiryDate.setDate(expiryDate.getDate() + 7);
-    const properties = {
-        id: user.id,
-        access_key: accessKey,
-        access_expires_at: expiryDate.toISOString()
-    };
-    csvUsersTxt += user.id + '\n';
-    const options = {
-        url: process.env["AC_ANALYTICS_BASE_URL"] + "anonUsers/" + process.env.AC_ANALYTICS_CLUSTER_ID + "/" + `${accessKey}${user.id}`,
-        headers: {
-            'X-API-KEY': process.env["AC_ANALYTICS_KEY"]
-        },
-        json: properties
-    };
-    request.post(options, (error) => {
-        done(error);
-    });
-};
-const collectAllPostsAndPoints = (activities, collectedPostIds, collectedPointIds) => {
-    activities.forEach((item) => {
-        if (item.Post) {
-            collectedPostIds.push(item.Post.id);
-        }
-        if (item.Point) {
-            collectedPointIds.push(item.Point.id);
-        }
-        if (!item.Point && !item.Post) {
-            log.warn("Can't find post or point id for activity in anonymous export", { item });
-        }
-    });
-};
-const sendAllCollectedPostAndPoints = (accessKey, collectedPostIds, collectedPointIds, encryptionKey, encryptionIv, done) => {
-    collectedPostIds = _.uniq(collectedPostIds);
-    collectedPointIds = _.uniq(collectedPointIds);
-    async.series([
-        (seriesCallback) => {
-            async.forEachSeries(collectedPostIds, (postId, forEachSeriesCallback) => {
-                models.Post.findOne({
-                    where: {
-                        id: postId
-                    },
-                    attributes: ['id', 'name', 'description', 'public_data', 'created_at', 'data']
-                }).then((post) => {
-                    let req = {
-                        query: {
-                            textType: 'postContent',
-                            targetLanguage: 'en'
-                        }
-                    };
-                    models.AcTranslationCache.getTranslation(req, post, function (error, descriptionEn) {
-                        req = {
-                            query: {
-                                textType: 'postName',
-                                targetLanguage: 'en'
-                            }
-                        };
-                        models.AcTranslationCache.getTranslation(req, post, function (error, nameEn) {
-                            saveAnonymousPost({
-                                id: getEncryptedId((post.id), encryptionKey, encryptionIv),
-                                name: post.name,
-                                name_en: nameEn ? nameEn.content : 'no translation',
-                                created_at: post.created_at,
-                                toxicity_score: (post.data && post.data.moderation) ? post.data.moderation.toxicityScore : null,
-                                description: post.description,
-                                description_en: descriptionEn ? descriptionEn.content : 'no translation',
-                            }, accessKey, forEachSeriesCallback);
-                        });
-                    });
-                }).catch((error) => {
-                    forEachSeriesCallback(error);
-                });
-            }, (error) => {
-                seriesCallback(error);
-            });
-        },
-        (seriesCallback) => {
-            async.forEachSeries(collectedPointIds, (pointId, forEachSeriesCallback) => {
-                models.Point.findOne({
-                    where: {
-                        id: pointId
-                    },
-                    order: [
-                        [models.PointRevision, 'created_at', 'asc'],
-                    ],
-                    attributes: ['id', 'created_at', 'content', 'value', 'data'],
-                    include: [
-                        {
-                            model: models.PointRevision,
-                            attributes: ['id', 'content']
-                        }
-                    ]
-                }).then((point) => {
-                    let req = {
-                        query: {
-                            textType: 'pointContent',
-                            targetLanguage: 'en'
-                        }
-                    };
-                    models.AcTranslationCache.getTranslation(req, point, function (error, contentEn) {
-                        saveAnonymousPoint({
-                            id: getEncryptedId((point.id), encryptionKey, encryptionIv),
-                            content: point.content,
-                            value: point.value,
-                            toxicity_score: (point.data && point.data.moderation) ? point.data.moderation.toxicityScore : null,
-                            created_at: point.created_at,
-                            content_en: contentEn ? contentEn.content : 'no translation'
-                        }, accessKey, forEachSeriesCallback);
-                    });
-                }).catch((error) => {
-                    forEachSeriesCallback(error);
-                });
-            }, (error) => {
-                seriesCallback(error);
-            });
-        },
-    ], (error) => {
-        done(error);
-    });
-};
-const sendAllUserActivitiesWithContent = (userId, communityId, collectedPostIds, collectedPointIds, encryptionKey, encryptionIv, accessKey, done) => {
-    models.AcActivity.findAll({
-        where: {
-            user_id: userId,
-            community_id: communityId,
-            type: {
-                $in: [
-                    "activity.post.new", "activity.post.opposition.new", "activity.post.endorsement.new",
-                    "activity.point.new", "activity.point.helpful.new", "activity.point.unhelpful.new"
-                ]
-            }
-        },
-        attributes: ['id', 'type', 'created_at', 'user_id', 'post_id', 'point_id'],
-        include: [
-            {
-                model: models.Post,
-                required: false,
-                attributes: ['id']
-            },
-            {
-                model: models.Point,
-                required: false,
-                attributes: ['id']
-            }
-        ]
-    }).then((activities) => {
-        collectAllPostsAndPoints(activities, collectedPostIds, collectedPointIds);
-        async.forEachSeries(activities, (activity, forEachSeriesCallback) => {
-            if (activity.Post || activity.Point) {
-                saveActivity({
-                    id: getEncryptedId(activity.id, encryptionKey, encryptionIv),
-                    type: activity.type,
-                    created_at: activity.created_at,
-                    post_id: activity.Post ? getEncryptedId(activity.Post.id, encryptionKey, encryptionIv) : null,
-                    point_id: activity.Point ? getEncryptedId(activity.Point.id, encryptionKey, encryptionIv) : null,
-                    user_id: getEncryptedId(activity.user_id, encryptionKey, encryptionIv)
-                }, accessKey, forEachSeriesCallback);
-            }
-            else {
-                log.warn("Can't find post or point for activity");
-                forEachSeriesCallback();
-            }
-        }, (error) => {
-            done(error);
-        });
-    }).catch(function (error) {
-        done(error);
-    });
-};
-const importCommunityUsersAndActivities = (communityId, accessKey, done) => {
-    const encryptionKey = crypto.randomBytes(32);
-    const encryptionIv = crypto.randomBytes(16);
-    let collectedPostIds = [];
-    let collectedPointIds = [];
-    models.Community.findOne({
-        where: {
-            id: communityId
-        },
-        include: [
-            {
-                model: models.User,
-                attributes: ['id'],
-                as: 'CommunityUsers',
-                required: true,
-            }
-        ]
-    }).then(function (community) {
-        if (community && community.CommunityUsers) {
-            async.forEachSeries(community.CommunityUsers, (user, eachSeriesCallback) => {
-                saveAnonymousUser({
-                    id: getEncryptedId(user.id, encryptionKey, encryptionIv)
-                }, accessKey, (error) => {
-                    if (error) {
-                        eachSeriesCallback(error);
-                    }
-                    else {
-                        sendAllUserActivitiesWithContent(user.id, communityId, collectedPostIds, collectedPointIds, encryptionKey, encryptionIv, accessKey, eachSeriesCallback);
-                    }
-                });
-            }, (error) => {
-                if (error) {
-                    done(error);
-                }
-                else {
-                    sendAllCollectedPostAndPoints(accessKey, collectedPostIds, collectedPointIds, encryptionKey, encryptionIv, (error) => {
-                        done(error);
-                    });
-                }
-            });
-        }
-        else {
-            done("Could not find community");
-        }
-    }).catch(function (error) {
-        done(error);
-    });
-};
-importCommunityUsersAndActivities(1017, "12345", () => {
-    fs.writeFileSync("/home/robert/Scratch/MMU/users.csv", csvUsersTxt);
-    fs.writeFileSync("/home/robert/Scratch/MMU/posts.csv", csvPostsTxt);
-    fs.writeFileSync("/home/robert/Scratch/MMU/points.csv", csvPointsTxt);
-    fs.writeFileSync("/home/robert/Scratch/MMU/activities.csv", csvActivitiesTxt);
-    process.exit();
-});
-module.exports = {
-    importCommunityUsersAndActivities
-};
-export {};

package/dist/active-citizen/engine/analytics/importer.js

@@ -1,308 +0,0 @@
-const models = require('../../../models/index.cjs');
-const _ = require('lodash');
-const async = require('async');
-const log = require('../../../utils/logger.cjs');
-const importDomain = require('./utils').importDomain;
-const importCommunity = require('./utils').importCommunity;
-const importGroup = require('./utils').importGroup;
-const importPost = require('./utils').importPost;
-const importPoint = require('./utils').importPoint;
-let updateAsyncLimit = 5;
-let lineCrCounter = 0;
-const processDots = () => {
-    if (lineCrCounter > 250) {
-        process.stdout.write("\n");
-        lineCrCounter = 1;
-    }
-    else {
-        process.stdout.write(".");
-        lineCrCounter += 1;
-    }
-};
-const importAllDomains = (done) => {
-    log.info('AcSimilarityDomainImport', {});
-    models.Domain.unscoped().findAll({
-        attributes: ['id', 'name', 'description', 'default_locale', 'created_at', 'updated_at'],
-        order: [
-            ['id', 'asc']
-        ]
-    }).then((domains) => {
-        lineCrCounter = 0;
-        async.eachOfLimit(domains, updateAsyncLimit, (domain, index, callback) => {
-            importDomain(domain, callback);
-        }, () => {
-            console.log("Finished updating domains");
-            done();
-        });
-    }).catch(function (error) {
-        done(error);
-    });
-};
-const importAllCommunities = (done) => {
-    log.info('AcSimilarityCommunityImport', {});
-    models.Community.unscoped().findAll({
-        include: [
-            {
-                model: models.Domain,
-                attributes: ['id', 'default_locale', 'created_at', 'updated_at'],
-                required: true
-            }
-        ],
-        attributes: ['id', 'name', 'description', 'default_locale', 'created_at', 'updated_at'],
-        order: [
-            ['id', 'asc']
-        ]
-    }).then((communities) => {
-        lineCrCounter = 0;
-        async.eachOfLimit(communities, updateAsyncLimit, (community, index, callback) => {
-            importCommunity(community, callback);
-        }, () => {
-            console.log("Finished updating communities");
-            done();
-        });
-    }).catch(function (error) {
-        done(error);
-    });
-};
-const importAllGroups = (done) => {
-    log.info('AcSimilarityGroupImport', {});
-    models.Group.unscoped().findAll({
-        include: [
-            {
-                model: models.Community,
-                attributes: ['id', 'access', 'status', 'default_locale', 'created_at', 'updated_at'],
-                required: true,
-                include: [
-                    {
-                        model: models.Domain,
-                        attributes: ['id', 'default_locale', 'created_at', 'updated_at'],
-                        required: true
-                    }
-                ]
-            }
-        ],
-        attributes: ['id', 'name', 'created_at', 'objectives', 'updated_at'],
-        order: [
-            ['id', 'asc']
-        ]
-    }).then((groups) => {
-        lineCrCounter = 0;
-        async.eachOfLimit(groups, updateAsyncLimit, (group, index, callback) => {
-            importGroup(group, callback);
-        }, () => {
-            console.log("Finished updating communities");
-            done();
-        });
-    }).catch(function (error) {
-        done(error);
-    });
-};
-const importAllPosts = (done) => {
-    log.info('AcSimilarityImport', {});
-    models.Post.unscoped().findAll({
-        include: [
-            {
-                model: models.Point,
-                required: false,
-                attributes: ['id', 'content'],
-            },
-            {
-                model: models.Group,
-                required: true,
-                attributes: ['id', 'access', 'status', 'configuration'],
-                include: [
-                    {
-                        attributes: ['id', 'formats'],
-                        model: models.Image, as: 'GroupLogoImages',
-                        required: false
-                    },
-                    {
-                        model: models.Community,
-                        attributes: ['id', 'access', 'status', 'default_locale'],
-                        required: true,
-                        include: [
-                            {
-                                attributes: ['id', 'formats'],
-                                model: models.Image, as: 'CommunityLogoImages',
-                                required: false
-                            },
-                            {
-                                model: models.Domain,
-                                attributes: ['id', 'default_locale'],
-                                required: true
-                            }
-                        ]
-                    }
-                ]
-            },
-            {
-                model: models.Image,
-                required: false,
-                as: 'PostHeaderImages',
-                attributes: ['id', 'formats']
-            },
-            {
-                model: models.Video,
-                required: false,
-                attributes: ['id', 'formats', 'updated_at', 'viewable', 'public_meta'],
-                as: 'PostVideos',
-                include: [
-                    {
-                        model: models.Image,
-                        as: 'VideoImages',
-                        attributes: ["formats", 'updated_at'],
-                        required: false
-                    },
-                ]
-            },
-            {
-                model: models.Audio,
-                required: false,
-                attributes: ['id', 'formats', 'updated_at', 'listenable'],
-                as: 'PostAudios',
-            }
-        ],
-        order: [
-            ['id', 'desc'],
-            [{ model: models.Image, as: 'PostHeaderImages' }, 'updated_at', 'asc'],
-            [{ model: models.Group }, { model: models.Image, as: 'GroupLogoImages' }, 'created_at', 'desc'],
-            [{ model: models.Group }, { model: models.Community }, { model: models.Image, as: 'CommunityLogoImages' }, 'created_at', 'desc']
-        ],
-        attributes: ['id', 'name', 'description', 'group_id', 'category_id', 'status', 'deleted', 'language', 'created_at', 'updated_at',
-            'user_id', 'official_status', 'public_data', 'cover_media_type',
-            'counter_endorsements_up', 'counter_endorsements_down', 'counter_points', 'counter_flags']
-    }).then((posts) => {
-        //posts = _.take(posts, 100);
-        lineCrCounter = 0;
-        async.eachOfLimit(posts, updateAsyncLimit, (post, index, callback) => {
-            importPost(post, callback);
-        }, () => {
-            console.log("Finished updating posts");
-            done();
-        });
-    }).catch(function (error) {
-        done(error);
-    });
-};
-const importAllPoints = (done) => {
-    models.Point.unscoped().findAll({
-        attributes: ['id', 'name', 'content', 'user_id', 'post_id', 'value', 'status', 'counter_quality_up', 'counter_quality_down', 'language', 'created_at', 'updated_at'],
-        order: [
-            ['id', 'desc'],
-            [models.PointRevision, 'created_at', 'asc'],
-            [{ model: models.Video, as: "PointVideos" }, 'updated_at', 'desc'],
-            [{ model: models.Audio, as: "PointAudios" }, 'updated_at', 'desc'],
-        ],
-        include: [
-            {
-                model: models.PointRevision,
-                attributes: ['content', 'value', 'created_at'],
-                required: false
-            },
-            {
-                model: models.Video,
-                required: false,
-                attributes: ['id', 'formats'],
-                as: 'PointVideos'
-            },
-            {
-                model: models.Audio,
-                required: false,
-                attributes: ['id', 'formats'],
-                as: 'PointAudios'
-            },
-            {
-                model: models.Post,
-                attributes: ['id', 'group_id', 'created_at', 'category_id', 'official_status', 'status', 'language'],
-                required: true,
-                include: [
-                    {
-                        model: models.Group,
-                        attributes: ['id', 'access', 'status', 'configuration'],
-                        required: true,
-                        include: [
-                            {
-                                model: models.Community,
-                                attributes: ['id', 'access', 'status', 'default_locale'],
-                                required: true,
-                                include: [
-                                    {
-                                        model: models.Domain,
-                                        attributes: ['id', 'default_locale'],
-                                        required: true
-                                    }
-                                ]
-                            }
-                        ]
-                    }
-                ]
-            }
-        ]
-    }).then(function (points) {
-        //points = _.take(points, 100);
-        lineCrCounter = 0;
-        async.eachOfLimit(points, updateAsyncLimit, (point, index, callback) => {
-            importPoint(point, callback);
-        }, () => {
-            console.log("Finished updating posts");
-            done();
-        });
-    }).catch(function (error) {
-        done(error);
-    });
-};
-const importAll = (done) => {
-    async.series([
-        (callback) => {
-            importAllDomains((error) => {
-                callback(error);
-            });
-        },
-        (callback) => {
-            importAllCommunities((error) => {
-                callback(error);
-            });
-        },
-        (callback) => {
-            importAllGroups((error) => {
-                callback(error);
-            });
-        },
-        (callback) => {
-            importAllPosts((error) => {
-                callback(error);
-            });
-        },
-        (callback) => {
-            importAllPoints((error) => {
-                callback(error);
-            });
-        }
-    ], (error) => {
-        console.log("Finished importing all");
-        if (error)
-            console.error(error);
-        done();
-    });
-};
-if (process.env["AC_ANALYTICS_KEY"] && process.env["AC_ANALYTICS_BASE_URL"]) {
-    log.info('AcSimilarityImportStarting', {});
-    if (process.argv[2] && process.argv[2] === "onlyUpdatePosts") {
-        updateAsyncLimit = 5;
-        importAllPosts(() => {
-            console.log("Done updating posts");
-            process.exit();
-        });
-    }
-    else {
-        importAll(() => {
-            console.log("Done importing all");
-            process.exit();
-        });
-    }
-}
-else {
-    console.error("NO AC_ANALYTICS_KEY");
-    process.exit();
-}
-export {};