@yrpri/api 9.0.101 → 9.0.103

package/dist/scripts/updateFromAlthingi.js

@@ -0,0 +1,303 @@
+var models = require('../models/index.cjs');
+var async = require('async');
+var _ = require('lodash');
+var http = require('http');
+//var parseString = require('xml2js').parseString;
+var concat = require('concat-stream');
+var SESSION_ID = process.argv[2]; // 146;
+var DOMAIN_ID = process.argv[3]; // 2
+var CRAWLER_USER_ID = process.argv[4]; // 29887
+var groupsConfigString = process.argv[5]; // "1=911:2=908:3=905:4=910:5=912:6=907:7=903:8=902:9=904:10=909:11=906"; //
+var createActivityEnabled = process.argv[6] && process.argv[6] == "activityEnabled";
+var baseIssueListURL = "http://www.althingi.is/altext/xml/thingmalalisti/?lthing=";
+var baseIssueURL = "http://www.althingi.is/altext/xml/thingmalalisti/thingmal/?lthing=";
+var USER_AGENT = 'Betra Ísland - Kæra Alþingi - Íbúar ses';
+var lawTopCategories = {
+    'Atvinnuvegir': 1,
+    'Erlend samskipti': 2,
+    'Hagstjórn': 3,
+    'Heilsa og heilbrigði': 4,
+    'Lög og réttur': 5,
+    'Mennta- og menningarmál': 6,
+    'Samfélagsmál': 7,
+    'Samgöngumál': 8,
+    'Stjórnarskipan og stjórnsýsla': 9,
+    'Trúmál og kirkja': 10,
+    'Umhverfismál': 11
+};
+var getJsonFromXml = function (url, callback) {
+    var opts = require('url').parse(url);
+    opts.headers = {
+        'User-Agent': USER_AGENT
+    };
+    http.get(opts, function (res) {
+        res.setEncoding('utf8');
+        res.pipe(concat(function (data) {
+            callback(null, JSON.parse(JSON.stringify(data)));
+            // Removed parseString
+            //parseString(data, function (err, result) {
+            // callback(null, JSON.parse(JSON.stringify(result)));
+            //});
+        }));
+    });
+};
+var getIssueList = function (callback) {
+    getJsonFromXml(baseIssueListURL + SESSION_ID, function (error, jsonResults) {
+        callback(error, jsonResults);
+    });
+};
+var topCategoryIdToGroup = {};
+var capitalize = function (string) {
+    string = string || '';
+    string = string.trim();
+    if (string[0]) {
+        string = string[0].toUpperCase() + string.substr(1).toLowerCase();
+    }
+    return string;
+};
+var setPostOfficialStatus = function (post, dbIssue) {
+    if (dbIssue.issueStatus) {
+        if (dbIssue.issueStatus.indexOf("Samþykkt") > -1) {
+            post.set('official_status', 2);
+        }
+        else if (dbIssue.issueStatus.indexOf("Fellt") > -1) {
+            post.set('official_status', -2);
+        }
+        else {
+            post.set('official_status', -1);
+        }
+    }
+};
+var addPost = function (dbIssue, userId, groupImageId, domain, callback) {
+    var post = models.Post.build({
+        name: dbIssue.name,
+        description: dbIssue.description,
+        group_id: dbIssue.groupId,
+        cover_media_type: groupImageId ? 'image' : 'none',
+        user_id: userId,
+        status: 'published',
+        counter_endorsements_up: 0,
+        content_type: models.Post.CONTENT_IDEA,
+        user_agent: USER_AGENT,
+        ip_address: '127.0.0.1'
+    });
+    post.set('public_data.law_issue', dbIssue);
+    setPostOfficialStatus(post, dbIssue);
+    post.save().then(function (post) {
+        async.series([
+            function (seriesCallback) {
+                models.PostRevision.build({
+                    name: post.name,
+                    description: post.description,
+                    group_id: post.group_id,
+                    user_id: userId,
+                    this_id: post.id,
+                    status: post.status,
+                    user_agent: post.user_agent,
+                    ip_address: post.ip_address
+                }).save().then(function () {
+                    post.updateAllExternalCounters({ ypDomain: domain }, 'up', 'counter_posts', function () {
+                        seriesCallback();
+                    });
+                }).catch(function (error) {
+                    seriesCallback(error);
+                });
+            },
+            function (seriesCallback) {
+                if (createActivityEnabled) {
+                    models.AcActivity.createActivity({
+                        type: 'activity.post.new',
+                        userId: post.user_id,
+                        domainId: domain.id,
+                        groupId: post.group_id,
+                        //                communityId: req.ypCommunity ?  req.ypCommunity.id : null,
+                        postId: post.id,
+                        access: models.AcActivity.ACCESS_PUBLIC
+                    }, function (error) {
+                        seriesCallback(error);
+                    });
+                }
+                else {
+                    seriesCallback();
+                }
+            },
+            function (seriesCallback) {
+                if (groupImageId) {
+                    models.Image.findOne({
+                        where: { id: groupImageId }
+                    }).then(function (image) {
+                        if (image) {
+                            post.addPostHeaderImage(image).then(function (error) {
+                                seriesCallback();
+                            });
+                        }
+                        else {
+                            seriesCallback();
+                        }
+                    });
+                }
+                else {
+                    seriesCallback();
+                }
+            }
+        ], function (error) {
+            callback(error);
+        });
+    }).catch(function (error) {
+        callback(error);
+    });
+};
+var saveIssueIfNeeded = function (dbIssue, userId, callback) {
+    models.Post.findOne({ where: {
+            $and: [
+                {
+                    "public_data.law_issue.sessionId": SESSION_ID
+                },
+                {
+                    "public_data.law_issue.issueId": dbIssue.issueId
+                }
+            ]
+        }
+    }).then(function (post) {
+        if (!post) {
+            models.Domain.findOne({
+                where: {
+                    id: DOMAIN_ID
+                }
+            }).then(function (domain) {
+                models.Group.findOne({
+                    where: {
+                        id: dbIssue.groupId
+                    }
+                }).then(function (group) {
+                    if (group) {
+                        var defaultImageId = null;
+                        if (group.configuration.defaultDataImageId) {
+                            defaultImageId = group.configuration.defaultDataImageId;
+                        }
+                        addPost(dbIssue, userId, defaultImageId, domain, callback);
+                    }
+                    else {
+                        console.error("Cant find group id: " + dbIssue.groupId);
+                        callback('Cant find group');
+                    }
+                });
+            });
+        }
+        else {
+            console.log("Already saved issue id: " + dbIssue.issueId);
+            if (post.public_data.law_issue.issueStatus != dbIssue.issueStatus) {
+                post.set('public_data.law_issue.issueStatus', dbIssue.issueStatus);
+                setPostOfficialStatus(post, dbIssue);
+                post.save().then(function (post) {
+                    callback();
+                });
+            }
+            else {
+                callback();
+            }
+        }
+    }).catch(function (error) {
+        callback(error);
+    });
+};
+var getIssueStatusFromVotes = function (votingSessions) {
+    if (!votingSessions || (votingSessions.length == 0 || votingSessions[0] == "")) {
+        return "Lagt fram";
+    }
+    else {
+        var statusValue = "";
+        if (votingSessions[0]['atkvæðagreiðsla'] && votingSessions[0]['atkvæðagreiðsla'].length > 0) {
+            _.each(votingSessions[0]['atkvæðagreiðsla'], function (votingSession) {
+                var tegund = votingSession['tegund'][0];
+                if (tegund == "Till.") {
+                    var resolution = votingSession['samantekt'][0]['afgreiðsla'][0];
+                    if (resolution == "samþykkt") {
+                        statusValue = "Samþykkt";
+                    }
+                    else {
+                        statusValue = "Fellt";
+                    }
+                }
+            });
+        }
+        else {
+            statusValue = "Í ferli";
+        }
+    }
+    return statusValue;
+};
+groupsConfigString.split(":").forEach(function (pair) {
+    var splitPair = pair.split("=");
+    topCategoryIdToGroup[splitPair[0]] = splitPair[1];
+});
+getIssueList(function (error, issueList) {
+    if (error) {
+        console.error(error);
+        process.exit();
+    }
+    else {
+        var issues = [];
+        async.eachSeries(issueList['málaskrá']['mál'], function (issue, callback) {
+            getJsonFromXml(issue['xml'][0], function (error, issueDetail) {
+                var dbIssue = { dataType: 'lawIssue', sessionId: SESSION_ID, issueId: issue.$['málsnúmer'], name: capitalize(issue['málsheiti'][0]),
+                    externalHtmlLink: issue['html'][0], xmlLink: issue['xml'][0], issueType: issue['málstegund'][0]['heiti'][0] };
+                if (error) {
+                    console.error(error);
+                    callback(error);
+                }
+                else {
+                    var topCategory, subCategory;
+                    if (issueDetail['þingmál'] && issueDetail['þingmál']['efnisflokkar']) {
+                        if (issueDetail['þingmál']['efnisflokkar'][0]['yfirflokkur'] && issueDetail['þingmál']['efnisflokkar'][0]['yfirflokkur'][1]) {
+                            topCategory = issueDetail['þingmál']['efnisflokkar'][0]['yfirflokkur'][1]['heiti'][0];
+                            subCategory = issueDetail['þingmál']['efnisflokkar'][0]['yfirflokkur'][1]['efnisflokkur'][0]['heiti'][0];
+                        }
+                        else if (issueDetail['þingmál']['efnisflokkar'][0]['yfirflokkur']) {
+                            topCategory = issueDetail['þingmál']['efnisflokkar'][0]['yfirflokkur'][0]['heiti'][0];
+                            subCategory = issueDetail['þingmál']['efnisflokkar'][0]['yfirflokkur'][0]['efnisflokkur'][0]['heiti'][0];
+                        }
+                        else if (SESSION_ID == 146 && (dbIssue.issueId == 1 || dbIssue.issueId == 2)) {
+                            topCategory = "Hagstjórn";
+                            subCategory = "Fjárreiður ríkisins";
+                        }
+                        if (topCategory) {
+                            var issueStatus = null;
+                            if (issueDetail['þingmál']['mál'][0]['staðamáls']) {
+                                issueStatus = issueDetail['þingmál']['mál'][0]['staðamáls'][0];
+                            }
+                            if (!issueStatus) {
+                                issueStatus = getIssueStatusFromVotes(issueDetail['þingmál']['atkvæðagreiðslur']);
+                            }
+                            dbIssue = _.merge(dbIssue, { topCategory: topCategory, subCategory: subCategory, issueStatus: issueStatus });
+                            dbIssue = _.merge(dbIssue, { topCategoryId: lawTopCategories[dbIssue.topCategory] });
+                            var description = capitalize(dbIssue.issueType) + ". " + dbIssue.name + ". " + dbIssue.topCategory + ". " + dbIssue.subCategory +
+                                ". Málið á Alþingi: " + dbIssue.externalHtmlLink;
+                            dbIssue = _.merge(dbIssue, { groupId: topCategoryIdToGroup[dbIssue.topCategoryId], description: description });
+                            if (capitalize(dbIssue.issueType).indexOf("Fyrirspurn") > -1 || capitalize(dbIssue.issueType).indexOf("Beiðni um skýrslu") > -1) {
+                                console.log("Not doing questions or reports for now for " + dbIssue.issueId);
+                                callback();
+                            }
+                            else {
+                                saveIssueIfNeeded(dbIssue, CRAWLER_USER_ID, callback);
+                            }
+                        }
+                        else {
+                            console.error("No topCategory");
+                            callback();
+                        }
+                    }
+                    else {
+                        console.error("No thingmal field");
+                        callback();
+                    }
+                }
+            });
+        }, function (error) {
+            if (error)
+                console.error(error);
+            process.exit();
+        });
+    }
+});
+export {};

package/dist/server.js

@@ -0,0 +1,3 @@
+import { YourPrioritiesApi } from './app.js';
+const app = new YourPrioritiesApi(8000);
+app.listen();

package/dist/utils/airbrake.cjs

@@ -0,0 +1,17 @@
+"use strict";
+if (process.env.AIRBRAKE_PROJECT_ID) {
+    const Airbrake = require('@airbrake/node');
+    let airBrake = null;
+    try {
+        airBrake = new Airbrake.Notifier({
+            projectId: process.env.AIRBRAKE_PROJECT_ID,
+            projectKey: process.env.AIRBRAKE_API_KEY,
+            performanceStats: false
+        });
+    }
+    catch (error) {
+        airBrake = null;
+        console.error(error);
+    }
+    module.exports = airBrake;
+}

package/dist/utils/cjsCodeReview.js

@@ -0,0 +1,99 @@
+import { OpenAI } from "openai";
+import * as fs from "fs";
+import * as path from "path";
+import { promisify } from "util";
+const readFilePromise = promisify(fs.readFile);
+export class YpCjsCodeReview {
+    constructor() {
+        this.modelName = "gpt-4o";
+        this.maxTokens = 4000;
+        this.temperature = 0.0;
+        this.openaiClient = new OpenAI({
+            apiKey: process.env.OPENAI_API_KEY,
+        });
+    }
+    async readFilesRecursively(dir) {
+        let results = [];
+        const list = fs.readdirSync(dir);
+        for (const file of list) {
+            if (file === 'node_modules')
+                continue;
+            if (file === 'ts-out')
+                continue;
+            const filePath = path.resolve(dir, file);
+            const stat = fs.statSync(filePath);
+            if (stat && stat.isDirectory()) {
+                results = results.concat(await this.readFilesRecursively(filePath));
+            }
+            else {
+                if (filePath.endsWith('.cjs')) {
+                    results.push(filePath);
+                }
+            }
+        }
+        return results;
+    }
+    async reviewCjsFiles() {
+        const files = await this.readFilesRecursively("./");
+        for (const file of files) {
+            console.log(`Reviewing file: ${file}`);
+            const content = await readFilePromise(file, "utf8");
+            const review = await this.callLlm(content);
+            console.log(`\n-----------------\nReview for file ${file}:\n${review}\n-----------------\n\n`);
+        }
+    }
+    renderSystemPrompt() {
+        return `You are an export Common module Javascript CJS code reviewer looking for any potential issues in the code that could cause a crash.
+    Instructions:
+    Review the code for all potential crash situations. Do not provide feedback on anything else than issues that could cause the code to crash.
+    If there are no crash related issues just output: No crash issues found.`;
+    }
+    renderUserMessage(codeToReview) {
+        return `Common module CJS Javascript code to review for crashes: ${codeToReview}
+
+Your review results:`;
+    }
+    async callLlm(codeToReview) {
+        const messages = [
+            {
+                role: "system",
+                content: this.renderSystemPrompt(),
+            },
+            {
+                role: "user",
+                content: this.renderUserMessage(codeToReview),
+            },
+        ];
+        const maxRetries = 3;
+        let retries = 0;
+        let running = true;
+        while (running) {
+            try {
+                //console.log(`Messages ${retries}:`, messages);
+                const results = await this.openaiClient.chat.completions.create({
+                    model: this.modelName,
+                    messages,
+                    max_tokens: this.maxTokens,
+                    temperature: this.temperature,
+                });
+                //console.log("Results:", results);
+                const textReview = results.choices[0].message.content;
+                return textReview;
+            }
+            catch (error) {
+                console.error("Error:", error);
+                retries++;
+                if (retries > maxRetries) {
+                    console.error("Max retries reached");
+                    running = false;
+                    return undefined;
+                }
+            }
+        }
+        return undefined;
+    }
+}
+(async () => {
+    const translator = new YpCjsCodeReview();
+    await translator.reviewCjsFiles();
+})();

package/dist/utils/community_mapping_tools.cjs

@@ -0,0 +1,124 @@
+"use strict";
+const models = require('../models/index.cjs');
+async function getTranslationForMap(textType, model, targetLanguage) {
+    return await new Promise((resolve, reject) => {
+        models.AcTranslationCache.getTranslation({ query: { textType, targetLanguage } }, model, async (error, translation) => {
+            if (!error && translation) {
+                resolve(translation.content);
+            }
+            else {
+                resolve();
+                console.log("No translation");
+            }
+        });
+    });
+}
+const truncate = (input, length, killwords, end) => {
+    length = length || 255;
+    if (input.length <= length)
+        return input;
+    if (killwords) {
+        input = input.substring(0, length);
+    }
+    else {
+        let idx = input.lastIndexOf(' ', length);
+        if (idx === -1) {
+            idx = length;
+        }
+        input = input.substring(0, idx);
+    }
+    input += (end !== undefined && end !== null) ? end : '...';
+    return input;
+};
+const getCommunityMap = async (communityId, map, options) => {
+    return await new Promise((resolve, reject) => {
+        models.Community.findOne({
+            where: {
+                id: communityId
+            },
+            attributes: ['id', 'name'],
+            include: [
+                {
+                    model: models.Group,
+                    attributes: ['id', 'name', 'configuration'],
+                    required: false
+                }
+            ]
+        }).then(async (community) => {
+            if (community) {
+                let communityName;
+                if (options.targetLocale) {
+                    communityName = await getTranslationForMap("communityName", community, "en");
+                }
+                else {
+                    communityName = community.name;
+                }
+                if (!communityName) {
+                    communityName = community.name;
+                }
+                const newCommunity = {
+                    name: truncate(communityName, 25) + ` (C-${community.id})`,
+                    link: "/community/" + community.id,
+                    type: "Community", children: []
+                };
+                for (const group of community.Groups) {
+                    let groupName;
+                    if (options.targetLocale) {
+                        groupName = await getTranslationForMap("groupName", group, "en");
+                    }
+                    else {
+                        groupName = group.name;
+                    }
+                    if (!groupName) {
+                        groupName = group.name;
+                    }
+                    groupName = truncate(groupName, 25) + ` (G-${group.id})`;
+                    if (group.configuration.actAsLinkToCommunityId) {
+                        await getCommunityMap(group.configuration.actAsLinkToCommunityId, newCommunity, options);
+                    }
+                    else {
+                        const newEntry = {
+                            name: groupName,
+                            link: "/group/" + group.id,
+                            children: []
+                        };
+                        newCommunity.children.push(newEntry);
+                    }
+                }
+                map.children.push(newCommunity);
+                resolve();
+            }
+            else {
+                console.error("Cant find community " + communityId);
+                map.children.push({ name: "Not found community id " + communityId, children: [] });
+                resolve();
+            }
+        }).catch(error => {
+            reject(error);
+        });
+    });
+};
+const getMapForCommunity = async (communityId, options) => {
+    return await new Promise((resolve, reject) => {
+        models.Community.findOne({
+            where: {
+                id: communityId
+            },
+            attributes: ['name', 'id']
+        }).then(async (community) => {
+            let map = { name: community.name, children: [] };
+            try {
+                await getCommunityMap(communityId, map, options);
+                resolve(map);
+            }
+            catch (error) {
+                reject(error);
+            }
+        }).catch(error => {
+            reject(error);
+        });
+    });
+};
+module.exports = {
+    getMapForCommunity
+};