@stamhoofd/backend-renderer 2.1.1

package/.env.template.json

@@ -0,0 +1,6 @@
+{
+    "environment": "development",
+    "PORT": 9093,
+    "CACHE_PATH": "/etc/stamhoofd/renderer/cache",
+    "INTERNAL_SECRET_KEY": ""
+}

package/.eslintrc.js

@@ -0,0 +1,61 @@
+module.exports = {
+    root: true,
+    ignorePatterns: ["dist/", "node_modules/"],
+    parserOptions: {
+        "ecmaVersion": 2017
+    },
+    env: {
+        "es6": true,
+        "node": true,
+    },
+    extends: [
+        "eslint:recommended",
+    ],
+    plugins: [],
+    rules: {
+        "no-console": "off",
+        "no-debugger": process.env.NODE_ENV === "production" ? "error" : "off",
+        "sort-imports": "off",
+        "import/order": "off"
+    },
+    overrides: [
+        {
+            // Rules for TypeScript and vue
+            files: ["*.ts"],
+            parser: "@typescript-eslint/parser",
+            parserOptions: {
+                project: ["./tsconfig.json"]
+            },
+            plugins: ["@typescript-eslint", "jest"],
+            extends: [
+                "eslint:recommended",
+                "plugin:@typescript-eslint/eslint-recommended",
+                "plugin:@typescript-eslint/recommended",
+                "plugin:@typescript-eslint/recommended-requiring-type-checking",
+                "plugin:jest/recommended",
+            ],
+            rules: {
+                "no-console": "off",
+                "no-debugger": process.env.NODE_ENV === "production" ? "error" : "off",
+                "sort-imports": "off",
+                "import/order": "off",
+                "@typescript-eslint/explicit-function-return-type": "off",
+                "@typescript-eslint/no-explicit-any": "off",
+                "@typescript-eslint/no-unused-vars": ["warn", { argsIgnorePattern: "^_" }],
+                "@typescript-eslint/no-namespace": "off",
+                "@typescript-eslint/no-floating-promises": "error",
+                "@typescript-eslint/no-misused-promises": "error",
+                "@typescript-eslint/prefer-for-of": "warn",
+                "@typescript-eslint/no-empty-interface": "off", // It is convenient to have placeholder interfaces
+                "@typescript-eslint/no-this-alias": "off", // No idea why we need this. This breaks code that is just fine. Prohibit the use of function() instead of this rule
+                "@typescript-eslint/unbound-method": "off", // Methods are automatically bound in vue, it would break removeEventListeners if we bound it every time unless we save every method in variables again...
+                "@typescript-eslint/no-unsafe-assignment": "off", // This is impossible to use with dependencies that don't have types yet, such as tiptap
+                "@typescript-eslint/no-unsafe-return": "off", // This is impossible to use with dependencies that don't have types yet, such as tiptap
+                "@typescript-eslint/no-unsafe-call": "off", // This is impossible to use with dependencies that don't have types yet, such as tiptap
+                "@typescript-eslint/no-unsafe-member-access": "off", // This is impossible to use with dependencies that don't have types yet, such as tiptap
+                "@typescript-eslint/restrict-plus-operands": "off", // bullshit one
+                "@typescript-eslint/explicit-module-boundary-types": "off",
+            },
+        }
+    ]
+};

package/README.md

@@ -0,0 +1 @@
+# Stamhoofd Renderer service

package/index.ts

@@ -0,0 +1,87 @@
+require('@stamhoofd/backend-env').load({service: 'renderer'});
+import { CORSPreflightEndpoint, Router, RouterServer } from "@simonbackx/simple-endpoints";
+import { I18n } from "@stamhoofd/backend-i18n";
+import { CORSMiddleware, LogMiddleware } from "@stamhoofd/backend-middleware";
+import { loadLogger } from "@stamhoofd/logging";
+
+process.on("unhandledRejection", (error: Error) => {
+    console.error("unhandledRejection");
+    console.error(error.message, error.stack);
+    process.exit(1);
+});
+
+// Set timezone!
+process.env.TZ = "UTC";
+
+// Quick check
+if (new Date().getTimezoneOffset() != 0) {
+    throw new Error("Process should always run in UTC timezone");
+}
+
+const start = async () => {
+    console.log('Started Renderer.')
+    loadLogger();
+    await I18n.load()
+    const router = new Router();
+    await router.loadEndpoints(__dirname + "/src/endpoints");
+    router.endpoints.push(new CORSPreflightEndpoint())
+
+    const routerServer = new RouterServer(router);
+    routerServer.verbose = false
+    
+    // Send the app version along
+    routerServer.addRequestMiddleware(LogMiddleware)
+    routerServer.addResponseMiddleware(LogMiddleware)
+
+    // Add CORS headers
+    routerServer.addResponseMiddleware(CORSMiddleware)
+
+    routerServer.listen(STAMHOOFD.PORT ?? 9090);
+
+    if (routerServer.server) {
+        // Default timeout is a bit too short
+        routerServer.server.timeout = 15000;
+    }
+
+    const shutdown = async () => {
+        console.log("Shutting down...")
+        // Disable keep alive
+        routerServer.defaultHeaders = Object.assign(routerServer.defaultHeaders, { 'Connection': 'close' })
+        if (routerServer.server) {
+            routerServer.server.headersTimeout = 5000;
+            routerServer.server.keepAliveTimeout = 1;
+        }
+
+        try {
+            await routerServer.close()
+            console.log("HTTP server stopped");
+        } catch (err) {
+            console.error("Failed to stop HTTP server:");
+            console.error(err);
+        }
+
+        // Should not be needed, but added for security as sometimes a promise hangs somewhere
+        process.exit(0);
+    };
+
+    process.on("SIGTERM", () => {
+        console.info("SIGTERM signal received.");
+        shutdown().catch((e) => {
+            console.error(e)
+            process.exit(1);
+        });
+    });
+
+    process.on("SIGINT", () => {
+        console.info("SIGINT signal received.");
+        shutdown().catch((e) => {
+            console.error(e)
+            process.exit(1);
+        });
+    });
+};
+
+start().catch(error => {
+    console.error("unhandledRejection", error);
+    process.exit(1);
+});

package/package.json

@@ -0,0 +1,34 @@
+{
+    "name": "@stamhoofd/backend-renderer",
+    "version": "2.1.1",
+    "main": "./dist/index.js",
+    "exports": {
+        ".": {
+            "require": "./dist/index.js"
+        }
+    },
+    "license": "UNLICENCED",
+    "scripts": {
+        "dev": "concurrently -r 'rm -rf ./dist && wait-on ./dist/index.js && nodemon --quiet --inspect=5860 --watch dist --exec node --enable-source-maps ./dist/index.js --signal SIGTERM' 'yarn build --watch --preserveWatchOutput > /dev/null'",
+        "build": "tsc -b",
+        "build:full": "yarn clear && yarn build",
+        "clear": "rm -rf ./dist",
+        "start": "yarn build && node --enable-source-maps ./dist/index.js",
+        "lint": "eslint . --ext .js,.jsx,.ts,.tsx"
+    },
+    "devDependencies": {
+        "@types/formidable": "3.4.5",
+        "@types/luxon": "^2.0.8",
+        "@types/mysql": "^2.15.20",
+        "@types/node": "^18.11.17"
+    },
+    "dependencies": {
+        "@simonbackx/simple-endpoints": "1.13.0",
+        "@simonbackx/simple-logging": "^1.0.1",
+        "formidable": "3.5.1",
+        "luxon": "^2.2.0",
+        "mockdate": "^3.0.2",
+        "mysql": "^2.18.1",
+        "puppeteer": "22.12.0"
+    }
+}

package/src/endpoints/HtmlToPdfEndpoint.ts

@@ -0,0 +1,214 @@
+import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
+import { SimpleError } from "@simonbackx/simple-errors";
+import { verifyInternalSignature } from "@stamhoofd/backend-env";
+import { QueueHandler } from "@stamhoofd/queues";
+import formidable from 'formidable';
+import { firstValues } from 'formidable/src/helpers/firstValues.js';
+import { promises as fs } from "fs";
+import puppeteer, { Browser } from "puppeteer";
+
+import { FileCache } from "../helpers/FileCache";
+
+type Params = Record<string, never>;
+type Body = undefined;
+type Query = undefined
+type ResponseBody = Buffer
+
+/**
+ * One endpoint to create, patch and delete groups. Usefull because on organization setup, we need to create multiple groups at once. Also, sometimes we need to link values and update multiple groups at once
+ */
+
+export class HtmlToPdfEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method != "POST") {
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, "/html-to-pdf", {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    async handle(request: DecodedRequest<Params, Query, Body>) {
+        const form = formidable({ 
+            maxTotalFileSize: 20 * 1024 * 1024, 
+            keepExtensions: false,
+            maxFiles: 1
+        });
+
+        const {html, cacheId, timestamp, signature} = await new Promise<{html: string, cacheId: string, timestamp: Date, signature: string}>((resolve, reject) => {
+            if (!request.request.request) {
+                reject(new SimpleError({
+                    code: "invalid_request",
+                    message: "Invalid request",
+                    statusCode: 500
+                }));
+                return;
+            }
+
+            // eslint-disable-next-line @typescript-eslint/no-misused-promises
+            form.parse(request.request.request, async (err, fieldsMultiple, files) => {
+                if (err) {
+                    reject(err);
+                    return;
+                }
+                if (!files.html || files.html.length !== 1) {
+                    reject(new SimpleError({
+                        code: "missing_field",
+                        message: "Field html is required",
+                        field: "html"
+                    }))
+                    return
+                }
+                const fields = firstValues(form, fieldsMultiple);
+
+                if (!fields.signature || typeof fields.signature !== "string") {
+                    reject(new SimpleError({
+                        code: "missing_field",
+                        message: "Field signature is required",
+                        field: "signature"
+                    }))
+                    return
+                }
+                if (!fields.cacheId || typeof fields.cacheId !== "string") {
+                    reject(new SimpleError({
+                        code: "missing_field",
+                        message: "Field cacheId is required",
+                        field: "cacheId"
+                    }))
+                    return
+                }
+                if (!fields.timestamp || typeof fields.timestamp !== "string") {
+                    reject(new SimpleError({
+                        code: "missing_field",
+                        message: "Field timestamp is required",
+                        field: "timestamp"
+                    }))
+                    return
+                }
+
+                let html;
+                try {
+                    html = await fs.readFile(files.html[0].filepath , 'utf8')
+                } catch (e) {
+                    reject(new SimpleError({
+                        code: "invalid_field",
+                        message: "Could not read html",
+                        field: "html"
+                    }))
+                    return
+                }
+
+                resolve({
+                    html,
+                    signature: fields.signature,
+                    cacheId: fields.cacheId,
+                    timestamp: new Date(parseInt(fields.timestamp as string))
+                })
+                return
+            });
+        });
+
+        // Verify signature first
+        if (!verifyInternalSignature(signature, cacheId, timestamp.getTime().toString(), html)) {
+            throw new SimpleError({
+                code: "invalid_signature",
+                message: "Invalid signature"
+            })
+        }
+
+        let pdf: Buffer | null = null
+        try {
+            pdf = await this.htmlToPdf(html, {retryCount: 2, startDate: new Date()})
+        } catch (e) {
+            console.error(e)
+        }
+        if (!pdf) {
+            throw new SimpleError({
+                code: "internal_error",
+                message: "Could not generate pdf"
+            })
+        }
+        await FileCache.write(cacheId, timestamp, pdf)
+        const response = new Response(pdf)
+        response.headers["Content-Type"] = "application/pdf"
+        response.headers["Content-Length"] = pdf.byteLength.toString()
+        return response;
+    }
+
+    browsers: ({browser: Browser, count: number}|null)[] = [null, null, null, null]
+    nextBrowserIndex = 0
+
+    async useBrowser<T>(callback: (browser: Browser) => Promise<T>): Promise<T> {
+        this.nextBrowserIndex++;
+        if (this.nextBrowserIndex >= this.browsers.length) {
+            this.nextBrowserIndex = 0;
+        }
+        return await QueueHandler.schedule("getBrowser" + this.nextBrowserIndex, async () => {
+            if (!this.browsers[this.nextBrowserIndex]) {
+                this.browsers[this.nextBrowserIndex] = { browser: await puppeteer.launch({ pipe: true }), count: 0 }
+            }
+            const browser = this.browsers[this.nextBrowserIndex]!
+            if (browser.count > 50 || !browser.browser.isConnected()) {
+                try {
+                    await browser.browser.close();
+                } catch (e) {
+                    console.error(e)
+                }
+                this.browsers[this.nextBrowserIndex] = { browser: await puppeteer.launch({ pipe: true }), count: 0 }
+            }
+            
+            return await callback(browser.browser)
+        });
+    }
+
+    async clearBrowser(browser: Browser) {
+        try {
+            await browser.close();
+        } catch (e) {
+            console.error(e)
+        }
+        const i = this.browsers.findIndex(b => b?.browser === browser)
+        if (i >= 0) {
+            this.browsers[i] = null
+        }
+    }
+
+    /**
+     * This will move to a different external service
+     */
+    async htmlToPdf(html: string, options: {retryCount: number, startDate: Date}): Promise<Buffer | null> {
+        const response = await this.useBrowser(async (browser) => {
+            try {
+                // Create a new page
+                const page = await browser.newPage();
+                await page.setJavaScriptEnabled(false);
+                await page.emulateMediaType('screen');
+                await page.setContent(html, { waitUntil: 'load' })
+
+                // Downlaod the PDF
+                const pdf = await page.pdf({
+                    // path: directory + this.id + '.pdf',
+                    margin: { top: '50px', right: '50px', bottom: '50px', left: '50px' },
+                    printBackground: true,
+                    format: 'A4',
+                    preferCSSPageSize: true,
+                    displayHeaderFooter: false
+                });
+                await page.close();
+                return pdf;
+            } catch (e) {
+                console.error('Failed to render document pdf', e)
+                return null;
+            }
+        })
+        if (response == null && options.retryCount > 0 && new Date().getTime() - options.startDate.getTime() < 15000) {
+            // Retry 
+            return await this.htmlToPdf(html, {...options, retryCount: options.retryCount - 1})
+        }
+        return response;
+    }
+}

package/src/endpoints/PdfCacheEndpoint.ts

@@ -0,0 +1,52 @@
+import { AutoEncoder, DateDecoder, Decoder, field, StringDecoder } from "@simonbackx/simple-encoding";
+import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
+import { SimpleError } from "@simonbackx/simple-errors";
+
+import { FileCache } from "../helpers/FileCache";
+
+type Params = Record<string, never>;
+class Query extends AutoEncoder {
+    @field({ decoder: StringDecoder })
+    cacheId: string
+
+    @field({ decoder: DateDecoder })
+    timestamp: Date
+}
+type Body = undefined
+type ResponseBody = Buffer
+
+/**
+ * One endpoint to create, patch and delete groups. Usefull because on organization setup, we need to create multiple groups at once. Also, sometimes we need to link values and update multiple groups at once
+ */
+
+export class PdfCacheEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    queryDecoder = Query as Decoder<Query>
+
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method != "GET") {
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, "/pdf-cache", {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    async handle(request: DecodedRequest<Params, Query, Body>) {
+        const pdf = await FileCache.read(request.query.cacheId, request.query.timestamp)
+        if (!pdf) {
+            throw new SimpleError({
+                code: "cache_not_found",
+                message: "Not cached",
+                statusCode: 404
+            })
+        }
+        const response = new Response(pdf)
+        response.headers["Content-Type"] = "application/pdf"
+        response.headers["Content-Length"] = pdf.byteLength.toString()
+        return response;
+    }
+}

package/src/helpers/FileCache.ts

@@ -0,0 +1,67 @@
+import { SimpleError } from '@simonbackx/simple-errors';
+import { promises as fs } from 'fs';
+
+export class FileCache {
+    static async write(cacheId: string, timestamp: Date, data: Buffer) {
+        if (cacheId.includes("/")) {
+            throw new SimpleError({
+                code: "invalid_field",
+                message: "Invalid cache id",
+                field: "cacheId"
+            })
+        }
+        
+        const folder = STAMHOOFD.CACHE_PATH + "/" + cacheId;
+        await fs.mkdir(folder, { recursive: true })
+
+        // Emtpy folder
+        const files = await fs.readdir(folder);
+        for (const file of files) {
+            const fileTimestamp = parseInt(file.substring(0, file.length - 4));
+            if (fileTimestamp <= timestamp.getTime()) {
+                await fs.unlink(folder + "/" + file);
+            }
+        }
+
+        const path = folder + "/" + timestamp.getTime() + ".pdf";
+        await fs.writeFile(path, data);
+    }
+
+    static async read(cacheId: string, timestamp: Date): Promise<Buffer | null> {
+        if (cacheId.includes("/")) {
+            throw new SimpleError({
+                code: "invalid_field",
+                message: "Invalid cache id",
+                field: "cacheId"
+            })
+        }
+
+        const folder = STAMHOOFD.CACHE_PATH + "/" + cacheId;
+        const path = folder + "/" + timestamp.getTime() + ".pdf";
+        try {
+            const data = await fs.readFile(path)
+            return data;
+        } catch {
+            // ignore
+        }
+
+        try {
+            const files = await fs.readdir(folder);
+            for (const file of files) {
+                const fileTimestamp = parseInt(file.substring(0, file.length - 4));
+                if (fileTimestamp >= timestamp.getTime()) {
+                    try {
+                        const data = await fs.readFile(folder + "/" + file)
+                        return data;
+                    } catch {
+                        // ignore
+                    }
+                }
+            }
+        } catch {
+            // ignore
+        }
+
+        return null;
+    }
+}

package/stamhoofd.d.ts

@@ -0,0 +1,15 @@
+
+export {};
+
+/**
+ * Stamhoofd uses a global variable to store some configurations. We don't use process.env because we can only store 
+ * strings into those files. And we need objects for our localized domains (different domains for each locale). 
+ * Having to encode and decode those values would be inefficient.
+ * 
+ * So we use our own global configuration variable: STAMHOOFD. Available everywhere and contains 
+ * other information depending on the environment (frontend/backend/shared). TypeScript will 
+ * always suggest the possible keys.
+ */
+declare global {
+    const STAMHOOFD: RendererEnvironment
+}

package/tsconfig.json

@@ -0,0 +1,35 @@
+{
+    "compilerOptions": {
+        "target": "es2019", // needs to be es2019 to support optional chaining. Node.js doesn't support optional chaining yet, so we need the transpiling
+        "module": "commonjs",
+        "jsx": "preserve",
+        "importHelpers": true,
+        "moduleResolution": "node",
+        "experimentalDecorators": true,
+        "esModuleInterop": true,
+        "allowSyntheticDefaultImports": true,
+        "sourceMap": true,
+        "strictNullChecks": true,
+        "declaration": true,
+        "declarationMap": true,
+        "outDir": "dist",
+        "lib": [
+            "es2019",
+            "ES2021.String",
+            "dom" // for puppeteer
+        ],
+        "types": [
+            "node",
+            "jest",
+            "@stamhoofd/backend-i18n",
+        ]
+    },
+    "include": [
+        "**/*.ts",
+        "../../../*.d.ts"
+    ],
+    "exclude": [
+        "node_modules",
+        "dist"
+    ]
+}