@stamhoofd/backend-renderer 2.119.0 → 2.120.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -7
- package/src/boot.ts +1 -1
- package/src/endpoints/HtmlToPdfEndpoint.ts +9 -7
- package/src/endpoints/PdfCacheEndpoint.ts +4 -2
- package/src/endpoints/PrerenderEndpoint.test.ts +102 -0
- package/src/endpoints/PrerenderEndpoint.ts +227 -0
- package/src/helpers/FileCache.ts +5 -0
- package/src/helpers/TTLFileCache.ts +91 -0
- package/{index.ts → src/index.ts} +1 -1
- package/stamhoofd.d.ts +2 -0
- package/tests/vitest.global.setup.ts +20 -0
- package/tests/vitest.setup.ts +31 -0
- package/tsconfig.build.json +15 -0
- package/tsconfig.json +9 -31
- package/tsconfig.test.json +17 -0
- package/vitest.config.js +13 -0
- package/eslint.config.mjs +0 -5
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@stamhoofd/backend-renderer",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.120.1",
|
|
4
|
+
"type": "module",
|
|
4
5
|
"main": "./dist/index.js",
|
|
5
6
|
"exports": {
|
|
6
7
|
".": {
|
|
@@ -10,12 +11,14 @@
|
|
|
10
11
|
"license": "UNLICENCED",
|
|
11
12
|
"scripts": {
|
|
12
13
|
"dev": "wait-on ../../shared/middleware/dist/index.js && concurrently -r 'yarn -s build --watch --preserveWatchOutput' \"wait-on ./dist/index.js && nodemon --quiet --inspect=5858 --watch dist --watch '../../../shared/*/dist/' --watch '../../shared/*/dist/' --ext .ts,.json,.sql,.js --watch .env.json --delay 1000ms --exec 'node --enable-source-maps ./dist/index.js' --signal SIGTERM\"",
|
|
14
|
+
"dev:build": "yarn -s build",
|
|
13
15
|
"dev:full": "yarn -s dev",
|
|
14
|
-
"build": "tsc
|
|
16
|
+
"build": "tsc --build tsconfig.build.json",
|
|
15
17
|
"build:full": "yarn -s clear && yarn -s build",
|
|
16
|
-
"clear": "rm -rf ./dist",
|
|
18
|
+
"clear": "rm -rf ./dist && rm -f *.tsbuildinfo",
|
|
17
19
|
"start": "yarn -s build && node --enable-source-maps ./dist/index.js",
|
|
18
|
-
"lint": "eslint"
|
|
20
|
+
"lint": "eslint",
|
|
21
|
+
"test": "vitest"
|
|
19
22
|
},
|
|
20
23
|
"devDependencies": {
|
|
21
24
|
"@types/formidable": "3.4.5",
|
|
@@ -24,16 +27,16 @@
|
|
|
24
27
|
"@types/node": "^22"
|
|
25
28
|
},
|
|
26
29
|
"dependencies": {
|
|
27
|
-
"@simonbackx/simple-endpoints": "1.
|
|
30
|
+
"@simonbackx/simple-endpoints": "1.21.0",
|
|
28
31
|
"@simonbackx/simple-logging": "^1.0.1",
|
|
29
32
|
"formidable": "3.5.4",
|
|
30
33
|
"luxon": "3.4.4",
|
|
31
34
|
"mockdate": "^3.0.2",
|
|
32
|
-
"mysql2": "^3.
|
|
35
|
+
"mysql2": "^3.20.0",
|
|
33
36
|
"puppeteer": "^24.11.0"
|
|
34
37
|
},
|
|
35
38
|
"publishConfig": {
|
|
36
39
|
"access": "public"
|
|
37
40
|
},
|
|
38
|
-
"gitHead": "
|
|
41
|
+
"gitHead": "00f65fc28d68feb86c30789784181a8954d638d7"
|
|
39
42
|
}
|
package/src/boot.ts
CHANGED
|
@@ -22,7 +22,7 @@ const start = async () => {
|
|
|
22
22
|
loadLogger();
|
|
23
23
|
await I18n.load();
|
|
24
24
|
const router = new Router();
|
|
25
|
-
await router.loadEndpoints(
|
|
25
|
+
await router.loadEndpoints(import.meta.dirname + '/endpoints');
|
|
26
26
|
router.endpoints.push(new CORSPreflightEndpoint());
|
|
27
27
|
|
|
28
28
|
const routerServer = new RouterServer(router);
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { DecodedRequest,
|
|
1
|
+
import type { DecodedRequest, Request} from '@simonbackx/simple-endpoints';
|
|
2
|
+
import { Endpoint, Response } from '@simonbackx/simple-endpoints';
|
|
2
3
|
import { SimpleError } from '@simonbackx/simple-errors';
|
|
3
4
|
import { verifyInternalSignature } from '@stamhoofd/backend-env';
|
|
4
5
|
import { QueueHandler } from '@stamhoofd/queues';
|
|
5
6
|
import formidable from 'formidable';
|
|
6
7
|
import { firstValues } from 'formidable/src/helpers/firstValues.js';
|
|
7
8
|
import { promises as fs } from 'fs';
|
|
8
|
-
import
|
|
9
|
+
import type { Browser } from 'puppeteer';
|
|
10
|
+
import puppeteer from 'puppeteer';
|
|
9
11
|
|
|
10
12
|
import { FileCache } from '../helpers/FileCache.js';
|
|
11
13
|
|
|
@@ -141,10 +143,10 @@ export class HtmlToPdfEndpoint extends Endpoint<Params, Query, Body, ResponseBod
|
|
|
141
143
|
return response;
|
|
142
144
|
}
|
|
143
145
|
|
|
144
|
-
browsers: ({ browser: Browser; count: number } | null)[] = [null, null, null, null];
|
|
145
|
-
nextBrowserIndex = 0;
|
|
146
|
+
static browsers: ({ browser: Browser; count: number } | null)[] = [null, null, null, null];
|
|
147
|
+
static nextBrowserIndex = 0;
|
|
146
148
|
|
|
147
|
-
async useBrowser<T>(callback: (browser: Browser) => Promise<T>): Promise<T> {
|
|
149
|
+
static async useBrowser<T>(callback: (browser: Browser) => Promise<T>): Promise<T> {
|
|
148
150
|
console.log('Requesting browser');
|
|
149
151
|
this.nextBrowserIndex++;
|
|
150
152
|
if (this.nextBrowserIndex >= this.browsers.length) {
|
|
@@ -169,7 +171,7 @@ export class HtmlToPdfEndpoint extends Endpoint<Params, Query, Body, ResponseBod
|
|
|
169
171
|
});
|
|
170
172
|
}
|
|
171
173
|
|
|
172
|
-
async clearBrowser(browser: Browser) {
|
|
174
|
+
static async clearBrowser(browser: Browser) {
|
|
173
175
|
try {
|
|
174
176
|
await browser.close();
|
|
175
177
|
}
|
|
@@ -186,7 +188,7 @@ export class HtmlToPdfEndpoint extends Endpoint<Params, Query, Body, ResponseBod
|
|
|
186
188
|
* This will move to a different external service
|
|
187
189
|
*/
|
|
188
190
|
async htmlToPdf(html: string, options: { retryCount: number; startDate: Date }): Promise<Uint8Array | null> {
|
|
189
|
-
const response = await
|
|
191
|
+
const response = await HtmlToPdfEndpoint.useBrowser(async (browser) => {
|
|
190
192
|
try {
|
|
191
193
|
// Create a new page
|
|
192
194
|
const page = await browser.newPage();
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { Decoder} from '@simonbackx/simple-encoding';
|
|
2
|
+
import { AutoEncoder, DateDecoder, field, StringDecoder } from '@simonbackx/simple-encoding';
|
|
3
|
+
import type { DecodedRequest, Request} from '@simonbackx/simple-endpoints';
|
|
4
|
+
import { Endpoint, Response } from '@simonbackx/simple-endpoints';
|
|
3
5
|
import { SimpleError } from '@simonbackx/simple-errors';
|
|
4
6
|
|
|
5
7
|
import { FileCache } from '../helpers/FileCache.js';
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { Request, TestServer } from '@simonbackx/simple-endpoints';
|
|
2
|
+
import { PrerenderEndpoint } from './PrerenderEndpoint.js';
|
|
3
|
+
import nock from 'nock';
|
|
4
|
+
import { STExpect } from '@stamhoofd/test-utils';
|
|
5
|
+
import fs from 'fs/promises';
|
|
6
|
+
|
|
7
|
+
async function fileExists(path: string): Promise<boolean> {
|
|
8
|
+
try {
|
|
9
|
+
await fs.access(path, fs.constants.F_OK);
|
|
10
|
+
return true;
|
|
11
|
+
} catch {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
describe('Endpoint.Prerender', () => {
|
|
17
|
+
const endpoint = new PrerenderEndpoint();
|
|
18
|
+
const testServer = new TestServer();
|
|
19
|
+
|
|
20
|
+
beforeAll(async () => {
|
|
21
|
+
await PrerenderEndpoint.fileCache.clear()
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
beforeEach(() => {
|
|
25
|
+
nock.enableNetConnect();
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
test('Returns 200 when healthy', async () => {
|
|
29
|
+
const request = Request.post({
|
|
30
|
+
path: '/prerender',
|
|
31
|
+
query: {
|
|
32
|
+
url: 'https://shop.stamhoofd.be/test/'
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const response = await testServer.test(endpoint, request);
|
|
37
|
+
expect(response.status).toBe(200);
|
|
38
|
+
expect(typeof response.body).toBe('string');
|
|
39
|
+
expect(response.body.startsWith('<!doctype html>')).toBe(true);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('Uses cache', async () => {
|
|
43
|
+
nock.disableNetConnect();
|
|
44
|
+
|
|
45
|
+
const request = Request.post({
|
|
46
|
+
path: '/prerender',
|
|
47
|
+
query: {
|
|
48
|
+
url: 'https://shop.stamhoofd.be/test/'
|
|
49
|
+
},
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
const response = await testServer.test(endpoint, request);
|
|
53
|
+
expect(response.status).toBe(200);
|
|
54
|
+
expect(typeof response.body).toBe('string');
|
|
55
|
+
expect(response.body.startsWith('<!doctype html>')).toBe(true);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('Does not use stale cache', async () => {
|
|
59
|
+
nock.disableNetConnect();
|
|
60
|
+
|
|
61
|
+
const file = PrerenderEndpoint.fileCache.keyToFilePath('https://shop.stamhoofd.be/test/');
|
|
62
|
+
const mtimeMs = Date.now() - 13 * 60 * 1000 * 60;
|
|
63
|
+
await fs.utimes(file, mtimeMs / 1000, mtimeMs / 1000);
|
|
64
|
+
|
|
65
|
+
const request = Request.post({
|
|
66
|
+
path: '/prerender',
|
|
67
|
+
query: {
|
|
68
|
+
url: 'https://shop.stamhoofd.be/test/'
|
|
69
|
+
},
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
await expect(testServer.test(endpoint, request)).rejects.toThrow(STExpect.simpleError({code: 'unavailable', statusCode: 503}))
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test('Purges stale cache', async () => {
|
|
76
|
+
nock.disableNetConnect();
|
|
77
|
+
|
|
78
|
+
const file = PrerenderEndpoint.fileCache.keyToFilePath('https://shop.stamhoofd.be/test/');
|
|
79
|
+
const mtimeMs = Date.now() - 13 * 60 * 1000 * 60;
|
|
80
|
+
await fs.utimes(file, mtimeMs / 1000, mtimeMs / 1000);
|
|
81
|
+
expect(await fileExists(file)).toEqual(true);
|
|
82
|
+
|
|
83
|
+
await PrerenderEndpoint.fileCache.purgeCache(12 * 60 * 1000 * 60);
|
|
84
|
+
|
|
85
|
+
expect(await fileExists(file)).toEqual(false);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('Returns 301 when redirecting to different domain locale', async () => {
|
|
89
|
+
const request = Request.post({
|
|
90
|
+
path: '/prerender',
|
|
91
|
+
query: {
|
|
92
|
+
url: 'https://shop.stamhoofd.nl/test/'
|
|
93
|
+
},
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
const response = await testServer.test(endpoint, request);
|
|
97
|
+
expect(response.status).toBe(301);
|
|
98
|
+
expect(response.headers).toEqual(expect.objectContaining({
|
|
99
|
+
location: 'https://shop.stamhoofd.be/test/'
|
|
100
|
+
}))
|
|
101
|
+
});
|
|
102
|
+
});
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import type { DecodedRequest, Request } from '@simonbackx/simple-endpoints';
|
|
2
|
+
import { Endpoint, Response } from '@simonbackx/simple-endpoints';
|
|
3
|
+
import { SimpleError } from '@simonbackx/simple-errors';
|
|
4
|
+
|
|
5
|
+
import type { Decoder } from '@simonbackx/simple-encoding';
|
|
6
|
+
import { AutoEncoder, field, URLDecoder } from '@simonbackx/simple-encoding';
|
|
7
|
+
import { TTLFileCache } from '../helpers/TTLFileCache.js';
|
|
8
|
+
import { HtmlToPdfEndpoint } from './HtmlToPdfEndpoint.js';
|
|
9
|
+
|
|
10
|
+
type Params = Record<string, never>;
|
|
11
|
+
type Body = undefined;
|
|
12
|
+
class Query extends AutoEncoder {
|
|
13
|
+
@field({decoder: new URLDecoder()})
|
|
14
|
+
url: URL
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
type Data = {
|
|
18
|
+
html: string;
|
|
19
|
+
headers: Record<string, string>;
|
|
20
|
+
statusCode: number
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
type ResponseBody = string;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* One endpoint to create, patch and delete groups. Usefull because on organization setup, we need to create multiple groups at once. Also, sometimes we need to link values and update multiple groups at once
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
export class PrerenderEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
|
|
30
|
+
queryDecoder = Query as Decoder<Query>
|
|
31
|
+
|
|
32
|
+
protected doesMatch(request: Request): [true, Params] | [false] {
|
|
33
|
+
if (request.method !== 'POST') {
|
|
34
|
+
return [false];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const params = Endpoint.parseParameters(request.url, '/prerender', {});
|
|
38
|
+
|
|
39
|
+
if (params) {
|
|
40
|
+
return [true, params as Params];
|
|
41
|
+
}
|
|
42
|
+
return [false];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async handle(request: DecodedRequest<Params, Query, Body>) {
|
|
46
|
+
const url = request.query.url.href
|
|
47
|
+
|
|
48
|
+
console.log('Prerendering ' + url)
|
|
49
|
+
|
|
50
|
+
let data: Data | null = null;
|
|
51
|
+
try {
|
|
52
|
+
data = await PrerenderEndpoint.getUrlHtml(url, { retryCount: 2, startDate: new Date() });
|
|
53
|
+
}
|
|
54
|
+
catch (e) {
|
|
55
|
+
console.error(e);
|
|
56
|
+
}
|
|
57
|
+
if (!data) {
|
|
58
|
+
throw new SimpleError({
|
|
59
|
+
statusCode: 503,
|
|
60
|
+
code: 'unavailable',
|
|
61
|
+
message: 'Could not prerender',
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
const response = new Response(data.html);
|
|
65
|
+
data.headers['content-type'] = 'text/html'
|
|
66
|
+
response.headers = data.headers
|
|
67
|
+
response.status = data.statusCode
|
|
68
|
+
return response;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
static readonly CACHE_TTL_MS = 60 * 1000 * 60 * 12; // 12 hours
|
|
72
|
+
static fileCache = new TTLFileCache('prerender', this.CACHE_TTL_MS)
|
|
73
|
+
|
|
74
|
+
static async getUrlHtml(url: string, options: { retryCount: number; startDate: Date }): Promise<Data | null> {
|
|
75
|
+
const existing = await this.fileCache.checkCache(url, this.CACHE_TTL_MS);
|
|
76
|
+
if (existing) {
|
|
77
|
+
try {
|
|
78
|
+
const d = JSON.parse(existing);
|
|
79
|
+
|
|
80
|
+
if (typeof d === 'object' && d !== null && 'html' in d && 'statusCode' in d && 'headers' in d) {
|
|
81
|
+
return d;
|
|
82
|
+
} else {
|
|
83
|
+
console.error('Invalid cached data point for ' + url)
|
|
84
|
+
}
|
|
85
|
+
} catch (e) {
|
|
86
|
+
console.error(e);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const response = await HtmlToPdfEndpoint.useBrowser(async (browser) => {
|
|
91
|
+
try {
|
|
92
|
+
const page = await browser.newPage();
|
|
93
|
+
|
|
94
|
+
await page.setUserAgent({
|
|
95
|
+
userAgent: 'prerender'
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
await page.setJavaScriptEnabled(true);
|
|
99
|
+
await page.emulateMediaType('screen');
|
|
100
|
+
|
|
101
|
+
// ── blockResources ────────────────────────────────────────────
|
|
102
|
+
// Block images, fonts, and media — we only need the HTML
|
|
103
|
+
await page.setRequestInterception(true);
|
|
104
|
+
const BLOCKED_RESOURCE_TYPES = new Set([
|
|
105
|
+
'image', 'media', 'font',
|
|
106
|
+
// Optionally also block these for extra speed:
|
|
107
|
+
'stylesheet', 'texttrack', 'object', 'beacon', 'csp_violationreport', 'imageset',
|
|
108
|
+
]);
|
|
109
|
+
page.on('request', (req) => {
|
|
110
|
+
if (BLOCKED_RESOURCE_TYPES.has(req.resourceType())) {
|
|
111
|
+
req.abort().catch(console.error);
|
|
112
|
+
} else {
|
|
113
|
+
req.continue().catch(console.error);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// Navigate and wait until network is quiet
|
|
118
|
+
const gotoResponse = await page.goto(url, {
|
|
119
|
+
waitUntil: 'networkidle0',
|
|
120
|
+
timeout: 30000,
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
// ── httpHeaders ───────────────────────────────────────────────
|
|
124
|
+
// Read prerender meta tags and honour status/header overrides.
|
|
125
|
+
// e.g. <meta name="prerender-status-code" content="404">
|
|
126
|
+
// <meta name="prerender-header" content="Location: /new-url">
|
|
127
|
+
const statusCode = gotoResponse?.status() ?? 200;
|
|
128
|
+
const metaStatusCode = await page
|
|
129
|
+
.evaluate(() => {
|
|
130
|
+
const el = document.querySelector('meta[name="prerender-status-code"]');
|
|
131
|
+
return el ? parseInt(el.getAttribute('content') ?? '', 10) : null;
|
|
132
|
+
})
|
|
133
|
+
.catch(() => null);
|
|
134
|
+
|
|
135
|
+
const headers = gotoResponse?.headers();
|
|
136
|
+
|
|
137
|
+
let effectiveStatus = metaStatusCode ?? statusCode;
|
|
138
|
+
// Surface the status so callers can act on it if needed
|
|
139
|
+
let metaLocation: string | undefined = undefined;
|
|
140
|
+
if (effectiveStatus === 301 || effectiveStatus === 302) {
|
|
141
|
+
const dmetaLocation = await page
|
|
142
|
+
.evaluate(() => {
|
|
143
|
+
const el = document.querySelector('meta[name="prerender-header"][content^="Location"]');
|
|
144
|
+
return el ? el.getAttribute('content')?.replace(/^Location:\s*/i, '') : null;
|
|
145
|
+
});
|
|
146
|
+
if (dmetaLocation) {
|
|
147
|
+
metaLocation = dmetaLocation
|
|
148
|
+
} else {
|
|
149
|
+
// something went wrong...
|
|
150
|
+
console.error('Missing Location header')
|
|
151
|
+
effectiveStatus = 500;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ── removeScriptTags + removePreloads ─────────────────────────
|
|
156
|
+
// Strip <script> tags and <link rel="preload|modulepreload"> from the DOM
|
|
157
|
+
await page.evaluate(() => {
|
|
158
|
+
// Remove all script tags
|
|
159
|
+
document.querySelectorAll('script').forEach((el) => el.remove());
|
|
160
|
+
|
|
161
|
+
// Remove preload / modulepreload link tags (removePreloads plugin)
|
|
162
|
+
document.querySelectorAll('link[rel="preload"][as="script"], link[rel="prefetch"][as="script"], link[rel="prefetch"][as="style"], link[rel="modulepreload"][as="script"]')
|
|
163
|
+
.forEach((el) => el.remove());
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
// Ibject <base href="url here" />
|
|
167
|
+
|
|
168
|
+
if (STAMHOOFD.environment === 'test') {
|
|
169
|
+
await page.evaluate(() => {
|
|
170
|
+
const child = document.createElement('base');
|
|
171
|
+
child.href = location.href;
|
|
172
|
+
document.head.prepend(child)
|
|
173
|
+
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const html = await page.evaluate(() => document.documentElement.outerHTML);
|
|
178
|
+
await page.close();
|
|
179
|
+
|
|
180
|
+
if (!headers) {
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// lowercase all headers
|
|
185
|
+
const cleanedHeaders: Record<string, string> = {};
|
|
186
|
+
for (const key of Object.keys(headers)) {
|
|
187
|
+
if (!headers[key]) {
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
const l = key.toLowerCase();
|
|
191
|
+
if (['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailer', 'transfer-encoding', 'upgrade', 'via', 'content-length', 'content-encoding', 'set-cookie', 'server'].includes(l)) {
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
cleanedHeaders[l] = headers[key]
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if (metaLocation) {
|
|
198
|
+
cleanedHeaders.location = metaLocation;
|
|
199
|
+
}
|
|
200
|
+
delete cleanedHeaders.connection;
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
html: '<!doctype html>' + html,
|
|
204
|
+
statusCode: effectiveStatus,
|
|
205
|
+
headers: cleanedHeaders
|
|
206
|
+
};
|
|
207
|
+
} catch (e) {
|
|
208
|
+
console.error('Failed to render url', e);
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
// Retry logic
|
|
214
|
+
if (response === null && options.retryCount > 0 && Date.now() - options.startDate.getTime() < 15000) {
|
|
215
|
+
return this.getUrlHtml(url, { ...options, retryCount: options.retryCount - 1 });
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Store in cache
|
|
219
|
+
if (response !== null) {
|
|
220
|
+
await this.fileCache.cacheFile(url, JSON.stringify(response))
|
|
221
|
+
return response;
|
|
222
|
+
|
|
223
|
+
}
|
|
224
|
+
return null;
|
|
225
|
+
|
|
226
|
+
}
|
|
227
|
+
}
|
package/src/helpers/FileCache.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { SimpleError } from '@simonbackx/simple-errors';
|
|
2
2
|
import { promises as fs } from 'fs';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* This cache does not use a TTL.
|
|
6
|
+
* Instead it uses the updatedAt timestamp of a resource when caching. So we keep the cache forever unless the resource itself has changed.
|
|
7
|
+
* The cache requester passes the updatedAt timestamp, so this works flawlessly.
|
|
8
|
+
*/
|
|
4
9
|
export class FileCache {
|
|
5
10
|
static async write(cacheId: string, timestamp: Date, data: Uint8Array | Buffer) {
|
|
6
11
|
if (cacheId.includes('/')) {
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { Formatter } from '@stamhoofd/utility';
|
|
2
|
+
import fs from 'fs/promises';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
|
|
5
|
+
export class TTLFileCache {
|
|
6
|
+
private cachePath: string;
|
|
7
|
+
|
|
8
|
+
constructor(name: string, ttl: number) {
|
|
9
|
+
if (!STAMHOOFD.CACHE_PATH) {
|
|
10
|
+
throw new Error('Missing STAMHOOFD.CACHE_PATH')
|
|
11
|
+
}
|
|
12
|
+
this.cachePath = STAMHOOFD.CACHE_PATH + '/' + Formatter.slug(name)
|
|
13
|
+
|
|
14
|
+
// Clear on boot
|
|
15
|
+
this.clear().catch(console.error)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
keyToFilePath(key: string): string {
|
|
19
|
+
// Percent-encode every character that isn't safe in a filename.
|
|
20
|
+
// Using encoding rather than replacement ensures the mapping is
|
|
21
|
+
// injective: no two distinct keys ever produce the same filename.
|
|
22
|
+
// Safe set: alphanumerics, hyphen, underscore, period.
|
|
23
|
+
// The percent sign itself is encoded as %25, so encoded output is
|
|
24
|
+
// always unambiguously decodable.
|
|
25
|
+
const safeKey = key.replace(/[^\w\-.]/g, (ch) => {
|
|
26
|
+
return '%' + ch.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0');
|
|
27
|
+
});
|
|
28
|
+
return path.join(this.cachePath, safeKey);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
async cacheFile(key: string, contents: string): Promise<void> {
|
|
33
|
+
await fs.mkdir(this.cachePath, { recursive: true });
|
|
34
|
+
await fs.writeFile(this.keyToFilePath(key), contents, 'utf-8');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async checkCache(key: string, ttlMs?: number): Promise<string | null> {
|
|
38
|
+
const filePath = this.keyToFilePath(key);
|
|
39
|
+
try {
|
|
40
|
+
if (ttlMs !== undefined) {
|
|
41
|
+
const stat = await fs.stat(filePath);
|
|
42
|
+
if (stat.mtimeMs < Date.now() - ttlMs) {
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return await fs.readFile(filePath, 'utf-8');
|
|
47
|
+
} catch (err: unknown) {
|
|
48
|
+
if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
throw err;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async clear() {
|
|
56
|
+
try {
|
|
57
|
+
await fs.rm(this.cachePath, {recursive: true})
|
|
58
|
+
} catch (e) {
|
|
59
|
+
// ignore
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async purgeCache(ttlMs: number): Promise<void> {
|
|
64
|
+
let entries: string[];
|
|
65
|
+
|
|
66
|
+
try {
|
|
67
|
+
entries = await fs.readdir(this.cachePath);
|
|
68
|
+
} catch (err: unknown) {
|
|
69
|
+
if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
|
|
70
|
+
return; // Cache directory doesn't exist yet — nothing to purge
|
|
71
|
+
}
|
|
72
|
+
throw err;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const cutoff = Date.now() - ttlMs;
|
|
76
|
+
|
|
77
|
+
await Promise.all(
|
|
78
|
+
entries.map(async (entry) => {
|
|
79
|
+
const filePath = path.join(this.cachePath, entry);
|
|
80
|
+
try {
|
|
81
|
+
const stat = await fs.stat(filePath);
|
|
82
|
+
if (stat.isFile() && stat.mtimeMs < cutoff) {
|
|
83
|
+
await fs.unlink(filePath);
|
|
84
|
+
}
|
|
85
|
+
} catch {
|
|
86
|
+
// File may have been deleted concurrently — ignore
|
|
87
|
+
}
|
|
88
|
+
})
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -5,7 +5,7 @@ backendEnv.load({ service: 'renderer' }).catch((error) => {
|
|
|
5
5
|
console.error('Failed to load environment:', error);
|
|
6
6
|
process.exit(1);
|
|
7
7
|
}).then(async () => {
|
|
8
|
-
await import('./
|
|
8
|
+
await import('./boot.js');
|
|
9
9
|
}).catch((error) => {
|
|
10
10
|
console.error('Failed to start the API:', error);
|
|
11
11
|
process.exit(1);
|
package/stamhoofd.d.ts
CHANGED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// first import nock
|
|
2
|
+
import nock from 'nock';
|
|
3
|
+
|
|
4
|
+
// prevent nock import from being removed on save
|
|
5
|
+
console.log('Imported nock: ', !!nock);
|
|
6
|
+
|
|
7
|
+
import { TestUtils } from '@stamhoofd/test-utils';
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
// Set timezone!
|
|
11
|
+
process.env.TZ = 'UTC';
|
|
12
|
+
|
|
13
|
+
// Quick check
|
|
14
|
+
if (new Date().getTimezoneOffset() !== 0) {
|
|
15
|
+
throw new Error('Process should always run in UTC timezone');
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export async function setup() {
|
|
19
|
+
TestUtils.globalSetup();
|
|
20
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// first import nock
|
|
2
|
+
|
|
3
|
+
import { Column } from '@simonbackx/simple-database';
|
|
4
|
+
import { Request } from '@simonbackx/simple-endpoints';
|
|
5
|
+
import { Version } from '@stamhoofd/structures';
|
|
6
|
+
import { TestUtils } from '@stamhoofd/test-utils';
|
|
7
|
+
|
|
8
|
+
Error.stackTraceLimit = 100;
|
|
9
|
+
|
|
10
|
+
// Set version of saved structures
|
|
11
|
+
Column.setJSONVersion(Version);
|
|
12
|
+
|
|
13
|
+
// Automatically set endpoint default version to latest one (only in tests!)
|
|
14
|
+
Request.defaultVersion = Version;
|
|
15
|
+
|
|
16
|
+
// Set timezone!
|
|
17
|
+
process.env.TZ = 'UTC';
|
|
18
|
+
|
|
19
|
+
// Quick check
|
|
20
|
+
if (new Date().getTimezoneOffset() !== 0) {
|
|
21
|
+
throw new Error('Process should always run in UTC timezone');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
console.log = () => {};
|
|
25
|
+
|
|
26
|
+
beforeAll(async () => {
|
|
27
|
+
// Override default $t handlers
|
|
28
|
+
TestUtils.loadEnvironment();
|
|
29
|
+
});
|
|
30
|
+
TestUtils.setPermanentEnvironment('CACHE_PATH', import.meta.dirname + '/../.test-cache')
|
|
31
|
+
TestUtils.setup();
|
package/tsconfig.json
CHANGED
|
@@ -1,34 +1,12 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
"sourceMap": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"declaration": true,
|
|
14
|
-
"declarationMap": true,
|
|
15
|
-
"outDir": "dist",
|
|
16
|
-
"lib": [
|
|
17
|
-
"es2022",
|
|
18
|
-
"dom" // for puppeteer
|
|
19
|
-
],
|
|
20
|
-
"types": [
|
|
21
|
-
"node",
|
|
22
|
-
"jest",
|
|
23
|
-
"@stamhoofd/backend-i18n",
|
|
24
|
-
]
|
|
25
|
-
},
|
|
26
|
-
"include": [
|
|
27
|
-
"**/*.ts",
|
|
28
|
-
"../../../*.d.ts"
|
|
29
|
-
],
|
|
30
|
-
"exclude": [
|
|
31
|
-
"node_modules",
|
|
32
|
-
"dist"
|
|
2
|
+
"extends": "../../../tsconfig.base.json",
|
|
3
|
+
"files": [],
|
|
4
|
+
"references": [
|
|
5
|
+
{
|
|
6
|
+
"path": "./tsconfig.build.json"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"path": "./tsconfig.test.json"
|
|
10
|
+
}
|
|
33
11
|
]
|
|
34
12
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "../../../tsconfig.test.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"rootDir": ".",
|
|
5
|
+
"outDir": "dist"
|
|
6
|
+
},
|
|
7
|
+
"references": [
|
|
8
|
+
{ "path": "./tsconfig.build.json" }
|
|
9
|
+
],
|
|
10
|
+
"include": [
|
|
11
|
+
"./stamhoofd.d.ts",
|
|
12
|
+
"../../../jest-extended.d.ts",
|
|
13
|
+
"./src/**/*.spec.ts",
|
|
14
|
+
"./src/**/*.test.ts",
|
|
15
|
+
"./tests"
|
|
16
|
+
]
|
|
17
|
+
}
|
package/vitest.config.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { defineConfig } from 'vitest/config';
|
|
2
|
+
|
|
3
|
+
export default defineConfig({
|
|
4
|
+
test: {
|
|
5
|
+
globalSetup: './tests/vitest.global.setup.ts',
|
|
6
|
+
setupFiles: ['./tests/vitest.setup.ts'],
|
|
7
|
+
watch: false,
|
|
8
|
+
globals: true,
|
|
9
|
+
root: import.meta.dirname,
|
|
10
|
+
isolate: true,
|
|
11
|
+
maxWorkers: 1, // For now we can't run parallel because all test files use the same database
|
|
12
|
+
},
|
|
13
|
+
});
|