@stamhoofd/backend-backup 2.50.2

package/index.ts

@@ -0,0 +1,108 @@
+import backendEnv from '@stamhoofd/backend-env';
+backendEnv.load({ service: 'backup' });
+
+import { CORSPreflightEndpoint, Router, RouterServer } from '@simonbackx/simple-endpoints';
+import { CORSMiddleware, LogMiddleware } from '@stamhoofd/backend-middleware';
+import { loadLogger } from '@stamhoofd/logging';
+import { startCrons, stopCrons, waitForCrons } from '@stamhoofd/crons';
+import { cleanBackups } from './src/helpers/backup';
+
+process.on('unhandledRejection', (error: Error) => {
+    console.error('unhandledRejection');
+    console.error(error.message, error.stack);
+    process.exit(1);
+});
+
+// Set timezone!
+process.env.TZ = 'UTC';
+
+// Quick check
+if (new Date().getTimezoneOffset() !== 0) {
+    throw new Error('Process should always run in UTC timezone');
+}
+
+const start = async () => {
+    console.log('Started Backup.');
+    loadLogger();
+    const router = new Router();
+    await router.loadEndpoints(__dirname + '/src/endpoints');
+    router.endpoints.push(new CORSPreflightEndpoint());
+
+    const routerServer = new RouterServer(router);
+    routerServer.verbose = false;
+
+    // Send the app version along
+    routerServer.addRequestMiddleware(LogMiddleware);
+    routerServer.addResponseMiddleware(LogMiddleware);
+
+    // Add CORS headers
+    routerServer.addResponseMiddleware(CORSMiddleware);
+
+    routerServer.listen(STAMHOOFD.PORT ?? 9090);
+
+    if (routerServer.server) {
+        // Default timeout is a bit too short
+        routerServer.server.timeout = 15000;
+    }
+
+    const shutdown = async () => {
+        console.log('Shutting down...');
+        // Disable keep alive
+        routerServer.defaultHeaders = Object.assign(routerServer.defaultHeaders, { Connection: 'close' });
+        if (routerServer.server) {
+            routerServer.server.headersTimeout = 5000;
+            routerServer.server.keepAliveTimeout = 1;
+        }
+
+        stopCrons();
+
+        if (STAMHOOFD.environment === 'development') {
+            setTimeout(() => {
+                console.error('Forcing exit after 5 seconds');
+                process.exit(1);
+            }, 5000);
+        }
+
+        try {
+            await routerServer.close();
+            console.log('HTTP server stopped');
+        }
+        catch (err) {
+            console.error('Failed to stop HTTP server:');
+            console.error(err);
+        }
+
+        await waitForCrons();
+
+        // Should not be needed, but added for security as sometimes a promise hangs somewhere
+        process.exit(0);
+    };
+
+    process.on('SIGTERM', () => {
+        console.info('SIGTERM signal received.');
+        shutdown().catch((e) => {
+            console.error(e);
+            process.exit(1);
+        });
+    });
+
+    process.on('SIGINT', () => {
+        console.info('SIGINT signal received.');
+        shutdown().catch((e) => {
+            console.error(e);
+            process.exit(1);
+        });
+    });
+
+    // Register crons
+    await import('./src/crons');
+    startCrons();
+
+    // Clean backups on boot (bit faster to retrieve the timestamp of the last backup for the health endpoint)
+    await cleanBackups();
+};
+
+start().catch((error) => {
+    console.error('unhandledRejection', error);
+    process.exit(1);
+});

package/package.json

@@ -0,0 +1,39 @@
+{
+    "name": "@stamhoofd/backend-backup",
+    "version": "2.50.2",
+    "main": "./dist/index.js",
+    "exports": {
+        ".": {
+            "require": "./dist/index.js"
+        }
+    },
+    "license": "UNLICENCED",
+    "scripts": {
+        "dev": "echo 'Waiting for shared backend packages...' && wait-on ../../shared/middleware/dist/index.js && echo 'Start building backend backup server' && concurrently -r 'yarn build --watch --preserveWatchOutput' \"wait-on ./dist/index.js && nodemon --quiet --inspect=5858 --watch dist --watch '../../../shared/*/dist/' --watch '../../shared/*/dist/' --ext .ts,.json,.sql,.js --watch .env.json --delay 1000ms --exec 'node --enable-source-maps ./dist/index.js' --signal SIGTERM\"",
+        "build": "tsc -b",
+        "build:full": "yarn clear && yarn build",
+        "clear": "rm -rf ./dist",
+        "start": "yarn build && node --enable-source-maps ./dist/index.js",
+        "lint": "eslint"
+    },
+    "devDependencies": {
+        "@types/formidable": "3.4.5",
+        "@types/luxon": "3.4.2",
+        "@types/mysql": "^2.15.20",
+        "@types/node": "^20.12"
+    },
+    "dependencies": {
+        "@aws-sdk/client-s3": "^3.678.0",
+        "@simonbackx/simple-endpoints": "1.14.0",
+        "@simonbackx/simple-logging": "^1.0.1",
+        "formidable": "3.5.1",
+        "luxon": "3.4.4",
+        "mockdate": "^3.0.2",
+        "mysql": "^2.18.1",
+        "puppeteer": "22.12.0"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "gitHead": "ffbc162ba4d0def7714d88b255b72a5f8991b4c3"
+}

package/src/crons.ts

@@ -0,0 +1,50 @@
+import { registerCron } from '@stamhoofd/crons';
+import { backup, backupBinlogs, cleanBackups, cleanBinaryLogBackups } from './helpers/backup';
+import { Formatter } from '@stamhoofd/utility';
+
+let lastFullBackup: Date | null = null;
+const backupStartTime = '02:00';
+const backupEndTime = '05:00';
+
+async function createBackups() {
+    const now = new Date();
+
+    if (lastFullBackup && Formatter.date(now, true) === Formatter.date(lastFullBackup, true)) {
+        return;
+    }
+
+    // Check time
+    if (Formatter.timeIso(now) < backupStartTime || Formatter.timeIso(now) > backupEndTime) {
+        return;
+    }
+
+    console.log('Creating full backup...');
+    await backup();
+
+    console.log('Full backup created');
+    lastFullBackup = now;
+}
+
+async function backupBinLogs() {
+    console.log('Backing up binlogs...');
+    await backupBinlogs();
+}
+
+let lastClean = new Date(0); // on boot
+const cleanInterval = 1000 * 60 * 60 * 4; // every 4 hours
+
+async function clean() {
+    const now = new Date();
+
+    if (now.getTime() - lastClean.getTime() > cleanInterval) {
+        console.log('Cleaning backups...');
+        await cleanBackups();
+        await cleanBinaryLogBackups();
+        lastClean = now;
+    }
+}
+
+registerCron('createBackups', createBackups);
+registerCron('backupBinLogs', backupBinLogs);
+
+registerCron('clean', clean);

package/src/endpoints/BackupEndpoint.ts

@@ -0,0 +1,59 @@
+import { AutoEncoder, field, StringDecoder } from '@simonbackx/simple-encoding';
+import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
+import { SimpleError } from '@simonbackx/simple-errors';
+import { backup } from '../helpers/backup';
+
+type Params = Record<string, never>;
+type Body = undefined;
+
+class Query extends AutoEncoder {
+    @field({ decoder: StringDecoder, optional: true })
+    key?: string;
+}
+
+type ResponseBody = string;
+
+/**
+ * One endpoint to create, patch and delete groups. Usefull because on organization setup, we need to create multiple groups at once. Also, sometimes we need to link values and update multiple groups at once
+ */
+
+export class BackupEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    queryDecoder = Query;
+
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method !== 'POST') {
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, '/backup', {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    async handle(request: DecodedRequest<Params, Query, Body>) {
+        if (!STAMHOOFD.HEALTH_ACCESS_KEY) {
+            throw new SimpleError({
+                code: 'unauthorized',
+                message: 'Unauthorized',
+                statusCode: 401,
+            });
+        }
+
+        if (STAMHOOFD.HEALTH_ACCESS_KEY && request.query.key !== STAMHOOFD.HEALTH_ACCESS_KEY) {
+            throw new SimpleError({
+                code: 'unauthorized',
+                message: 'Unauthorized',
+                statusCode: 401,
+            });
+        }
+
+        backup().catch(console.error);
+
+        const response = new Response('Scheduled backup');
+        response.status = 201;
+        return response;
+    }
+}

package/src/endpoints/HealthEndpoint.ts

@@ -0,0 +1,83 @@
+import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
+import { BackupHealth, getHealth } from '../helpers/backup';
+import { AutoEncoder, field, StringDecoder } from '@simonbackx/simple-encoding';
+import { SimpleError } from '@simonbackx/simple-errors';
+import { checkReplicaStatus } from '../helpers/replica-status';
+
+type Params = Record<string, never>;
+type Body = undefined;
+
+class Query extends AutoEncoder {
+    @field({ decoder: StringDecoder, optional: true })
+    key?: string;
+}
+
+export class BackupWithReplicationHealth extends BackupHealth {
+    @field({ decoder: StringDecoder, nullable: true })
+    replicationError: string | null = null;
+}
+
+type ResponseBody = BackupWithReplicationHealth;
+
+/**
+ * One endpoint to create, patch and delete groups. Usefull because on organization setup, we need to create multiple groups at once. Also, sometimes we need to link values and update multiple groups at once
+ */
+
+export class HealthEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    queryDecoder = Query;
+
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method !== 'GET') {
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, '/health', {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    async handle(request: DecodedRequest<Params, Query, Body>) {
+        if (!STAMHOOFD.HEALTH_ACCESS_KEY) {
+            throw new SimpleError({
+                code: 'unauthorized',
+                message: 'Unauthorized',
+                statusCode: 401,
+            });
+        }
+
+        if (STAMHOOFD.HEALTH_ACCESS_KEY && request.query.key !== STAMHOOFD.HEALTH_ACCESS_KEY) {
+            throw new SimpleError({
+                code: 'unauthorized',
+                message: 'Unauthorized',
+                statusCode: 401,
+            });
+        }
+
+        const baseHealth = getHealth();
+        const health = BackupWithReplicationHealth.create({
+            ...baseHealth,
+        });
+
+        if (STAMHOOFD.IS_REPLICA) {
+            try {
+                await checkReplicaStatus();
+            }
+            catch (e) {
+                health.replicationError = e.message;
+                health.status = 'error';
+            }
+        }
+
+        const response = new Response(
+            health,
+        );
+
+        if (health.status === 'error') {
+            response.status = 503;
+        }
+        return response;
+    }
+}