domma-cms 1.61.0 → 1.61.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/CLAUDE.md +1 -1
- package/docs/DommaFeatures.md +3 -1
- package/docs/cache.md +24 -0
- package/docs/configuration.md +1 -1
- package/package.json +1 -1
- package/public/js/effects.js +1 -1
- package/public/js/site.js +1 -1
- package/server/config.js +37 -0
- package/server/routes/api/chats.js +3 -1
- package/server/routes/api/components.js +2 -1
- package/server/routes/api/content-api.js +1 -1
- package/server/routes/api/content-public.js +2 -2
- package/server/routes/api/content-writes.js +1 -1
- package/server/routes/api/effects.js +4 -9
- package/server/routes/api/layouts.js +1 -1
- package/server/routes/api/pages.js +6 -5
- package/server/routes/api/search.js +3 -3
- package/server/routes/api/sidebar-counts.js +2 -0
- package/server/routes/api/theme.js +2 -1
- package/server/routes/api/views.js +2 -1
- package/server/routes/public.js +4 -4
- package/server/server.js +12 -6
- package/server/services/cache/adminChange.js +28 -0
- package/server/services/content.js +73 -10
- package/server/services/effectsSettings.js +25 -0
- package/server/services/health.js +2 -2
- package/server/services/htmlResponse.js +33 -4
- package/server/services/listTags.js +1 -1
- package/server/services/markdown.js +56 -22
- package/server/services/pluginGuides.js +2 -2
- package/server/services/plugins.js +25 -2
- package/server/services/projects.js +39 -4
- package/server/services/publicAssets.js +488 -0
- package/server/services/renderer.js +10 -8
- package/server/services/seo/index.js +3 -3
- package/server/services/sitemap.js +1 -1
- package/server/services/versions.js +2 -2
- package/server/services/wpImport/index.js +2 -2
- package/server/templates/page.html +3 -2
|
@@ -110,7 +110,8 @@ export async function pagesRoutes(fastify) {
|
|
|
110
110
|
const canDelete = {preHandler: [authenticate, requirePermission('pages', 'delete')]};
|
|
111
111
|
|
|
112
112
|
// Render markdown preview (shortcodes + sanitize, no frontmatter)
|
|
113
|
-
|
|
113
|
+
// The two previews render and return; the editor calls them as you type. Not a change (cache/adminChange.js).
|
|
114
|
+
fastify.post('/pages/preview', {...canRead, config: {keepsCache: true}}, async (request, reply) => {
|
|
114
115
|
const {markdown} = request.body;
|
|
115
116
|
if (typeof markdown !== 'string') return reply.status(400).send({error: 'markdown must be a string'});
|
|
116
117
|
const {html} = await parseMarkdown(`---\ntitle: preview\n---\n${markdown}`, {user: request.user || null});
|
|
@@ -125,7 +126,7 @@ export async function pagesRoutes(fastify) {
|
|
|
125
126
|
// theme, menus, dconfig and custom CSS are all exactly what shipping
|
|
126
127
|
// would produce. No draft banner: the editor is already telling the user
|
|
127
128
|
// what they are looking at.
|
|
128
|
-
fastify.post('/pages/preview/full', canRead, async (request, reply) => {
|
|
129
|
+
fastify.post('/pages/preview/full', {...canRead, config: {keepsCache: true}}, async (request, reply) => {
|
|
129
130
|
const {frontmatter = {}, body = '', urlPath = '/'} = request.body || {};
|
|
130
131
|
if (typeof body !== 'string') return reply.status(400).send({error: 'body must be a string'});
|
|
131
132
|
if (typeof urlPath !== 'string' || !urlPath.startsWith('/')) {
|
|
@@ -199,7 +200,7 @@ export async function pagesRoutes(fastify) {
|
|
|
199
200
|
|
|
200
201
|
// Aggregate unique tags from all pages - must be registered before the wildcard /pages/* route
|
|
201
202
|
fastify.get('/pages/tags', canRead, async (request, reply) => {
|
|
202
|
-
const pages = (await listPages()).filter(p => inScope(request.user, p));
|
|
203
|
+
const pages = (await listPages({render: false})).filter(p => inScope(request.user, p));
|
|
203
204
|
const tagSet = new Set();
|
|
204
205
|
pages.forEach(p => {
|
|
205
206
|
if (Array.isArray(p.tags)) p.tags.forEach(t => { if (t) tagSet.add(t); });
|
|
@@ -209,7 +210,7 @@ export async function pagesRoutes(fastify) {
|
|
|
209
210
|
|
|
210
211
|
// The sidebar badge. Static, so it is matched before the /pages/* wildcard below.
|
|
211
212
|
fastify.get('/pages/_count', canRead, async (request) =>
|
|
212
|
-
pagesBadge((await listPages()).filter(p => inScope(request.user, p))));
|
|
213
|
+
pagesBadge((await listPages({render: false})).filter(p => inScope(request.user, p))));
|
|
213
214
|
|
|
214
215
|
// Domma's option types for the editor (admin/js/lib/dconfig-check.js). Static, before /pages/*.
|
|
215
216
|
fastify.get('/pages/_domma-catalogue', canRead, async (request, reply) => {
|
|
@@ -220,7 +221,7 @@ export async function pagesRoutes(fastify) {
|
|
|
220
221
|
|
|
221
222
|
// List all pages
|
|
222
223
|
fastify.get('/pages', canRead, async (request, reply) => {
|
|
223
|
-
const pages = (await listPages()).filter(p => inScope(request.user, p));
|
|
224
|
+
const pages = (await listPages({render: false})).filter(p => inScope(request.user, p));
|
|
224
225
|
const counts = await Promise.all(pages.map(p => getVersionCount(p.urlPath)));
|
|
225
226
|
return pages.map((p, i) => ({
|
|
226
227
|
urlPath: p.urlPath,
|
|
@@ -57,7 +57,7 @@ export async function searchRoutes(fastify, opts = {}) {
|
|
|
57
57
|
// no-store (the /api onSend hook) and never enter the response cache,
|
|
58
58
|
// so nothing gated can be replayed to another viewer.
|
|
59
59
|
const viewer = await resolveViewer(request);
|
|
60
|
-
const pages = await listPages();
|
|
60
|
+
const pages = await listPages({render: false});
|
|
61
61
|
const results = searchPages(pages, q, {
|
|
62
62
|
maxResults: settings.maxResults,
|
|
63
63
|
viewer,
|
|
@@ -81,7 +81,7 @@ export async function searchRoutes(fastify, opts = {}) {
|
|
|
81
81
|
const settings = getSearchSettings();
|
|
82
82
|
const asked = parseInt(request.query.max, 10);
|
|
83
83
|
const maxResults = Number.isFinite(asked) ? Math.min(50, Math.max(1, asked)) : settings.maxResults;
|
|
84
|
-
const pages = await listPages();
|
|
84
|
+
const pages = await listPages({render: false});
|
|
85
85
|
const exclusions = await searchExclusions(pages);
|
|
86
86
|
const results = searchPages(pages, q, {
|
|
87
87
|
maxResults,
|
|
@@ -92,7 +92,7 @@ export async function searchRoutes(fastify, opts = {}) {
|
|
|
92
92
|
});
|
|
93
93
|
|
|
94
94
|
fastify.get('/search/_count', canReadBadge, async () => {
|
|
95
|
-
const pages = await listPages();
|
|
95
|
+
const pages = await listPages({render: false});
|
|
96
96
|
return searchBadge(getSearchSettings(), pages, await searchExclusions(pages));
|
|
97
97
|
});
|
|
98
98
|
}
|
|
@@ -8,6 +8,8 @@ export async function sidebarCountRoutes(fastify, opts = {}) {
|
|
|
8
8
|
const isItemAllowed = opts.isItemAllowed || (() => true);
|
|
9
9
|
const timeoutMs = opts.countTimeoutMs || 5000;
|
|
10
10
|
fastify.post('/sidebar/counts', {
|
|
11
|
+
// A read that happens to be a POST, polled by every open admin: not a change (cache/adminChange.js).
|
|
12
|
+
config: {keepsCache: true},
|
|
11
13
|
preHandler: [authenticate], bodyLimit: 32 * 1024,
|
|
12
14
|
schema: {body: {type: 'object', additionalProperties: false, properties: {
|
|
13
15
|
urls: {type: 'array', maxItems: 256, uniqueItems: true, items: {type: 'string', maxLength: 512}},
|
|
@@ -142,7 +142,8 @@ export async function themeRoutes(fastify, opts = {}) {
|
|
|
142
142
|
}
|
|
143
143
|
});
|
|
144
144
|
|
|
145
|
-
|
|
145
|
+
// Compiles what it is sent and returns it: not a change (cache/adminChange.js).
|
|
146
|
+
fastify.post('/theme/compile', {...canRead, config: {keepsCache: true}}, async (request, reply) => {
|
|
146
147
|
const candidate = request.body || {};
|
|
147
148
|
const errors = validateThemeConfig(candidate, {remembered: rememberedIds(await loadThemeConfig())});
|
|
148
149
|
if (errors.length) return reply.code(400).send({error: errors.join('; '), validation: errors});
|
|
@@ -181,7 +181,8 @@ export async function viewsRoutes(fastify) {
|
|
|
181
181
|
// user may see (project scope), and it runs as an admin preview - no
|
|
182
182
|
// row-level filter, the same as executing a saved view from the admin.
|
|
183
183
|
// `_` is outside the slug alphabet, so no view can shadow the route.
|
|
184
|
-
|
|
184
|
+
// Runs an unsaved view and returns its rows: not a change (cache/adminChange.js).
|
|
185
|
+
fastify.post('/views/_preview', {...canUpdate, config: {keepsCache: true}}, async (request, reply) => {
|
|
185
186
|
const {source, connection = 'default', stages = [], limit} = request.body || {};
|
|
186
187
|
if (typeof source !== 'string' || !source) return reply.status(400).send({error: 'Choose a source collection'});
|
|
187
188
|
if (!Array.isArray(stages)) return reply.status(400).send({error: 'stages must be an array'});
|
package/server/routes/public.js
CHANGED
|
@@ -17,7 +17,7 @@ import {validSessionCookie, newSessionCookie} from '../services/shortcodeExpiry.
|
|
|
17
17
|
import {touchPreviewLink, verifyPreviewToken} from '../services/previewLinks.js';
|
|
18
18
|
import {viewerRoleKey} from '../services/userRoles.js';
|
|
19
19
|
import {hooks} from '../services/hooks.js';
|
|
20
|
-
import {
|
|
20
|
+
import {readConfig} from '../config.js';
|
|
21
21
|
import * as cache from '../services/cache/index.js';
|
|
22
22
|
|
|
23
23
|
/**
|
|
@@ -34,7 +34,7 @@ import * as cache from '../services/cache/index.js';
|
|
|
34
34
|
* @returns {string} e.g. 'https://example.com' (no trailing slash)
|
|
35
35
|
*/
|
|
36
36
|
export function getBaseUrl(request) {
|
|
37
|
-
const override = (
|
|
37
|
+
const override = (readConfig('site') || {}).baseUrl;
|
|
38
38
|
if (override) return override.toString().trim().replace(/\/+$/, '');
|
|
39
39
|
const host = request.host || request.headers.host || request.hostname;
|
|
40
40
|
return `${request.protocol}://${host}`;
|
|
@@ -283,7 +283,7 @@ export async function publicRoutes(fastify) {
|
|
|
283
283
|
const pageMtime = await getPageMtime(page.urlPath || urlPath);
|
|
284
284
|
const cacheKey = `page:${urlPath}:m${pageMtime}:role${viewerRoleKey(userObj)}:o${baseUrl}`;
|
|
285
285
|
// The footer's blocks (site.json footer.blocks) are on every page: editing one must reach the cached pages.
|
|
286
|
-
const cacheTags = [`page:${urlPath}`, ...(page.cacheTags || []), 'nav', 'site', ...footerBlockCacheTags(
|
|
286
|
+
const cacheTags = [`page:${urlPath}`, ...(page.cacheTags || []), 'nav', 'site', ...footerBlockCacheTags(readConfig('site'))];
|
|
287
287
|
// x-dm-cache says which way this response went - hit, miss, off (the
|
|
288
288
|
// cache is disabled) or bypass (a draft, an expiring or a per-viewer
|
|
289
289
|
// page, never cached) - so a slow page can be read from the browser.
|
|
@@ -294,7 +294,7 @@ export async function publicRoutes(fastify) {
|
|
|
294
294
|
reply.header('x-dm-cache', status);
|
|
295
295
|
hooks.emit('content:pageViewed', {urlPath, title: page.title || ''});
|
|
296
296
|
// cached: this HTML is shared, so its compressed body is kept too (htmlResponse.js).
|
|
297
|
-
return sendHtml(request, reply, html, {cached: true});
|
|
297
|
+
return sendHtml(request, reply, html, {cached: true, key: cacheKey});
|
|
298
298
|
});
|
|
299
299
|
}
|
|
300
300
|
|
package/server/server.js
CHANGED
|
@@ -19,7 +19,7 @@ import path from 'path';
|
|
|
19
19
|
import fs from 'fs/promises';
|
|
20
20
|
import {fileURLToPath} from 'url';
|
|
21
21
|
import {createRequire} from 'module';
|
|
22
|
-
import {config, getConfig} from './config.js';
|
|
22
|
+
import {config, getConfig, readConfig} from './config.js';
|
|
23
23
|
import {getLoadedPlugins, getPluginSettings, recordPluginLoadFailure, registerPlugins} from './services/plugins.js';
|
|
24
24
|
import {notifierFor} from './services/notify.js';
|
|
25
25
|
import {grantAdminToolDefaults, grantPermissionOnce, load as loadRoles, seed as seedRoles} from './services/roles.js';
|
|
@@ -34,6 +34,7 @@ import {parseTrustProxy} from './services/network.js';
|
|
|
34
34
|
import {requestSerializer} from './services/request-log.js';
|
|
35
35
|
import {rateLimitOptions} from './services/rate-limit.js';
|
|
36
36
|
import {registerAdminAssets} from './services/adminAssets.js';
|
|
37
|
+
import {registerPublicAssets} from './services/publicAssets.js';
|
|
37
38
|
import {isActiveType, servedType} from './services/mediaTypes.js';
|
|
38
39
|
import {connectSources, frameSources, imageSources} from './services/embedCsp.js';
|
|
39
40
|
import {cspSources as gaSources} from './services/googleAnalytics.js';
|
|
@@ -131,7 +132,7 @@ await app.register(helmet, {
|
|
|
131
132
|
scriptSrc: ["'self'", "'unsafe-inline'", "'wasm-unsafe-eval'", 'https://cdn.jsdelivr.net', gaSources('script')],
|
|
132
133
|
styleSrc: ["'self'", "'unsafe-inline'", 'https://cdn.jsdelivr.net', 'https://fonts.googleapis.com'],
|
|
133
134
|
// config/server.json csp.imgHosts: other sites a page may show images from (services/embedCsp.js).
|
|
134
|
-
imgSrc: [() => imageSources(
|
|
135
|
+
imgSrc: [() => imageSources(readConfig('server').csp?.imgHosts).join(' '), gaSources('img')],
|
|
135
136
|
fontSrc: ["'self'", 'data:', 'https://fonts.gstatic.com'],
|
|
136
137
|
// config/server.json csp.connectHosts: other sites a page may read from (services/embedCsp.js).
|
|
137
138
|
connectSrc: [...connectSources(serverConfig.csp?.connectHosts), gaSources('connect')],
|
|
@@ -399,11 +400,11 @@ try {
|
|
|
399
400
|
* tag. Runs on onResponse, after the reply has gone, so it costs the caller
|
|
400
401
|
* nothing - and a memory-driver clear is two Map resets.
|
|
401
402
|
*/
|
|
402
|
-
|
|
403
|
+
// A route that changes nothing a page could show opts out with `config: {keepsCache: true}`
|
|
404
|
+
// (services/cache/adminChange.js) - the admin's 30-second chat check-in was emptying the cache.
|
|
405
|
+
const {isCacheClearingChange} = await import('./services/cache/adminChange.js');
|
|
403
406
|
app.addHook('onResponse', async (request, reply) => {
|
|
404
|
-
if (!
|
|
405
|
-
if (reply.statusCode >= 400) return;
|
|
406
|
-
if (!request.user) return;
|
|
407
|
+
if (!isCacheClearingChange(request, reply)) return;
|
|
407
408
|
try {
|
|
408
409
|
await cache.clear();
|
|
409
410
|
} catch (err) {
|
|
@@ -442,6 +443,10 @@ await app.register(staticPlugin, {
|
|
|
442
443
|
}
|
|
443
444
|
});
|
|
444
445
|
|
|
446
|
+
// Fingerprinted public assets are cached for good, and the site's custom CSS is
|
|
447
|
+
// served as a file - services/publicAssets.js.
|
|
448
|
+
const publicAssets = registerPublicAssets(app, {root: ROOT, dommaDist: DOMMA_DIST, pluginsDir});
|
|
449
|
+
|
|
445
450
|
// Serve plugin admin/ and public/ subdirs only - block plugin.js, config.js, data/, etc.
|
|
446
451
|
await app.register(async function pluginStaticScope(instance) {
|
|
447
452
|
instance.addHook('onRequest', (request, reply, done) => {
|
|
@@ -792,6 +797,7 @@ await registerPlugins(app);
|
|
|
792
797
|
}
|
|
793
798
|
// After the plugins' onRequest hooks - see serve() in services/adminAssets.js.
|
|
794
799
|
adminAssets.serve();
|
|
800
|
+
publicAssets.serve();
|
|
795
801
|
|
|
796
802
|
// ---------------------------------------------------------------------------
|
|
797
803
|
// Public Plugin Routes (root-level, before catch-all)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does this finished request count as an admin change that clears the
|
|
3
|
+
* response cache? (The safety net in server.js - see the comment there.)
|
|
4
|
+
*
|
|
5
|
+
* A signed-in, successful POST/PUT/PATCH/DELETE does - unless its route says
|
|
6
|
+
* it changes nothing a page could show: `config: {keepsCache: true}`. The
|
|
7
|
+
* admin's "available for chats" check-in is one, sent every 30 seconds from
|
|
8
|
+
* every admin screen; counted as a change, one signed-in person emptied the
|
|
9
|
+
* whole cache twice a minute and no page was ever served from it.
|
|
10
|
+
*
|
|
11
|
+
* Only for a route whose effect lives outside every rendered page. When in
|
|
12
|
+
* doubt leave it off: a cache cleared too often is slow, a cache not cleared
|
|
13
|
+
* is wrong.
|
|
14
|
+
*/
|
|
15
|
+
const CLEARING_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {{method: string, user?: object, routeOptions?: {config?: {keepsCache?: boolean}}}} request
|
|
19
|
+
* @param {{statusCode: number}} reply
|
|
20
|
+
* @returns {boolean}
|
|
21
|
+
*/
|
|
22
|
+
export function isCacheClearingChange(request, reply) {
|
|
23
|
+
if (!CLEARING_METHODS.has(request.method)) return false;
|
|
24
|
+
if (reply.statusCode >= 400) return false;
|
|
25
|
+
if (!request.user) return false;
|
|
26
|
+
if (request.routeOptions?.config?.keepsCache === true) return false;
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
@@ -35,6 +35,9 @@ const MEDIA_DIR = config.content.mediaDir;
|
|
|
35
35
|
// the hooks do not know about. Entries are cloned in and out, so a caller
|
|
36
36
|
// that edits the object it was handed edits its own copy.
|
|
37
37
|
const PAGE_MEMO_MAX = 500;
|
|
38
|
+
/** listPages() raises the limit to the number of pages, up to this. */
|
|
39
|
+
const PAGE_MEMO_CEILING = 5000;
|
|
40
|
+
let _pageMemoMax = PAGE_MEMO_MAX;
|
|
38
41
|
const PAGE_MEMO_TTL_MS = 5 * 60 * 1000;
|
|
39
42
|
/** @type {Map<string, {mtimeMs: number, size: number, at: number, page: object}>} */
|
|
40
43
|
const _pageMemo = new Map();
|
|
@@ -43,11 +46,13 @@ cache.onInvalidate(() => _pageMemo.clear());
|
|
|
43
46
|
|
|
44
47
|
/** For the cache status route and tests: how the memo is doing. */
|
|
45
48
|
export function pageMemoStats() {
|
|
46
|
-
return {size: _pageMemo.size, max:
|
|
49
|
+
return {size: _pageMemo.size, max: _pageMemoMax, hits: _pageMemoStats.hits, misses: _pageMemoStats.misses};
|
|
47
50
|
}
|
|
48
51
|
|
|
49
52
|
export function _resetPageMemoForTests() {
|
|
50
53
|
_pageMemo.clear();
|
|
54
|
+
_summaryMemo.clear();
|
|
55
|
+
_pageMemoMax = PAGE_MEMO_MAX;
|
|
51
56
|
_pageMemoStats.hits = 0;
|
|
52
57
|
_pageMemoStats.misses = 0;
|
|
53
58
|
}
|
|
@@ -69,7 +74,7 @@ async function readPageMemo(filePath) {
|
|
|
69
74
|
} catch {
|
|
70
75
|
return page; // something in it cannot be cloned: hand it over unmemoised
|
|
71
76
|
}
|
|
72
|
-
while (_pageMemo.size >
|
|
77
|
+
while (_pageMemo.size > _pageMemoMax) _pageMemo.delete(_pageMemo.keys().next().value);
|
|
73
78
|
return page;
|
|
74
79
|
}
|
|
75
80
|
|
|
@@ -77,21 +82,80 @@ async function readPageMemo(filePath) {
|
|
|
77
82
|
// Pages
|
|
78
83
|
// ---------------------------------------------------------------------------
|
|
79
84
|
|
|
85
|
+
// ---------------------------------------------------------------------------
|
|
86
|
+
// Page summaries (front matter + source, nothing rendered)
|
|
87
|
+
// ---------------------------------------------------------------------------
|
|
88
|
+
//
|
|
89
|
+
// A page's front matter and Markdown source depend on its file and nothing
|
|
90
|
+
// else, so they are kept per file and checked against the file's mtime and
|
|
91
|
+
// size on every read - no TTL, and correct with the response cache on or off.
|
|
92
|
+
// Emptied with the response cache all the same, so two saves inside one mtime
|
|
93
|
+
// tick cannot leave the first one's values behind. Cloned out, so a caller
|
|
94
|
+
// that edits a row edits its own copy.
|
|
95
|
+
/** @type {Map<string, {mtimeMs: number, size: number, page: object}>} */
|
|
96
|
+
const _summaryMemo = new Map();
|
|
97
|
+
cache.onInvalidate(() => _summaryMemo.clear());
|
|
98
|
+
|
|
99
|
+
async function readPageSummary(filePath) {
|
|
100
|
+
const st = await fs.stat(filePath);
|
|
101
|
+
const kept = _summaryMemo.get(filePath);
|
|
102
|
+
if (kept && kept.mtimeMs === st.mtimeMs && kept.size === st.size) return structuredClone(kept.page);
|
|
103
|
+
const raw = await fs.readFile(filePath, 'utf8');
|
|
104
|
+
// Options passed on purpose: without them gray-matter keeps every string it
|
|
105
|
+
// has ever parsed (and hands the same `data` object to every caller).
|
|
106
|
+
const {data, content} = matter(raw, {});
|
|
107
|
+
const page = {...data, urlPath: filePathToUrlPath(filePath), content};
|
|
108
|
+
try {
|
|
109
|
+
_summaryMemo.set(filePath, {mtimeMs: st.mtimeMs, size: st.size, page: structuredClone(page)});
|
|
110
|
+
} catch { /* something in the front matter cannot be cloned: read it each time */ }
|
|
111
|
+
return page;
|
|
112
|
+
}
|
|
113
|
+
|
|
80
114
|
/**
|
|
81
115
|
* Recursively list all .md files under PAGES_DIR.
|
|
82
|
-
* Returns
|
|
116
|
+
* Returns page objects sorted by sortOrder then title.
|
|
117
|
+
*
|
|
118
|
+
* By default every page is parsed in full, as getPage() does - shortcodes,
|
|
119
|
+
* collections, sanitising - which is tens of milliseconds a page and seconds
|
|
120
|
+
* for a few hundred. A caller that reads only the front matter (title, status,
|
|
121
|
+
* layout, tags, seo, ...) or the Markdown source (`content`) asks for
|
|
122
|
+
* `{render: false}` and gets exactly those plus `urlPath` and
|
|
123
|
+
* `resolvedProject`: no `html`, `usedComponents`, `cacheTags`, `perViewer` or
|
|
124
|
+
* `hasExpiry`. Lists, badges, search and the sitemap all do.
|
|
125
|
+
*
|
|
126
|
+
* The rendered rows come through the parsed-page memo while the response
|
|
127
|
+
* cache is on, so a second listing does not parse every page again.
|
|
83
128
|
*
|
|
129
|
+
* @param {{render?: boolean}} [opts]
|
|
84
130
|
* @returns {Promise<object[]>}
|
|
85
131
|
*/
|
|
86
|
-
export async function listPages() {
|
|
132
|
+
export async function listPages({render = true} = {}) {
|
|
87
133
|
const files = await collectMdFiles(PAGES_DIR);
|
|
88
|
-
|
|
134
|
+
let pages;
|
|
135
|
+
if (!render) {
|
|
136
|
+
pages = await Promise.all(files.map(filePath => readPageSummary(filePath)));
|
|
137
|
+
// Forget pages that are gone, so the memo holds the site's pages and no more.
|
|
138
|
+
if (_summaryMemo.size > files.length) {
|
|
139
|
+
const live = new Set(files);
|
|
140
|
+
for (const key of _summaryMemo.keys()) if (!live.has(key)) _summaryMemo.delete(key);
|
|
141
|
+
}
|
|
142
|
+
} else if (cache.isEnabled()) {
|
|
143
|
+
// Every page must fit at once, or a listing evicts what it is about to read.
|
|
144
|
+
_pageMemoMax = Math.max(_pageMemoMax, Math.min(PAGE_MEMO_CEILING, files.length + 50));
|
|
145
|
+
pages = await Promise.all(files.map(filePath => readPageMemo(filePath)));
|
|
146
|
+
} else {
|
|
147
|
+
pages = await Promise.all(files.map(filePath => readPageFile(filePath)));
|
|
148
|
+
}
|
|
89
149
|
pages.sort((a, b) => (a.sortOrder ?? 99) - (b.sortOrder ?? 99) || a.title.localeCompare(b.title));
|
|
90
|
-
|
|
150
|
+
// One read of the projects for the whole list, not one per page.
|
|
151
|
+
const {listProjects, projectForPage} = await import('./projects.js');
|
|
152
|
+
let projects = null;
|
|
91
153
|
for (const page of pages) {
|
|
92
154
|
const url = page.url || page.urlPath || '/';
|
|
93
155
|
const explicit = (page.project === undefined) ? undefined : page.project;
|
|
94
|
-
page.resolvedProject =
|
|
156
|
+
if (typeof explicit === 'string' && explicit) { page.resolvedProject = explicit; continue; }
|
|
157
|
+
projects ??= await listProjects();
|
|
158
|
+
page.resolvedProject = projectForPage(projects, url, explicit);
|
|
95
159
|
}
|
|
96
160
|
return pages;
|
|
97
161
|
}
|
|
@@ -327,9 +391,8 @@ export async function setPageFrontmatter(urlPath, patch) {
|
|
|
327
391
|
throw err;
|
|
328
392
|
}
|
|
329
393
|
const raw = await fs.readFile(filePath, 'utf8');
|
|
330
|
-
// gray-matter
|
|
331
|
-
|
|
332
|
-
const parsed = matter(raw);
|
|
394
|
+
// Options passed so gray-matter does not keep the string (see readPageSummary).
|
|
395
|
+
const parsed = matter(raw, {});
|
|
333
396
|
const data = {...parsed.data};
|
|
334
397
|
for (const [k, v] of Object.entries(patch || {})) {
|
|
335
398
|
if (v === undefined) delete data[k];
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The scroll-reveal defaults every public page's effects read
|
|
3
|
+
* (config/effects.json over the built-in defaults).
|
|
4
|
+
*
|
|
5
|
+
* Its own module so both the route that serves them and the renderer - which
|
|
6
|
+
* writes them into the page, saving every page view a request before its
|
|
7
|
+
* effects can start - read the same thing.
|
|
8
|
+
*/
|
|
9
|
+
import {getConfig} from '../config.js';
|
|
10
|
+
|
|
11
|
+
export const EFFECTS_DEFAULTS = {respectMotion: true, defaultDuration: 600, defaultAnimation: 'fade', defaultThreshold: 0.1};
|
|
12
|
+
|
|
13
|
+
/** @returns {object} the defaults with config/effects.json over them */
|
|
14
|
+
export function loadEffectsSettings() {
|
|
15
|
+
try {
|
|
16
|
+
return {...EFFECTS_DEFAULTS, ...(getConfig('effects') || {})};
|
|
17
|
+
} catch {
|
|
18
|
+
return {...EFFECTS_DEFAULTS};
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** `window.__CMS_EFFECTS__ = {...};` for a page's inline script (public/js/effects.js reads it). */
|
|
23
|
+
export function effectsSettingsScript() {
|
|
24
|
+
return `window.__CMS_EFFECTS__ = ${JSON.stringify(loadEffectsSettings()).replace(/</g, '\\u003c')};`;
|
|
25
|
+
}
|
|
@@ -125,7 +125,7 @@ async function checkContentDisk() {
|
|
|
125
125
|
}
|
|
126
126
|
|
|
127
127
|
async function checkStalePages() {
|
|
128
|
-
const pages = await listPages().catch(() => []);
|
|
128
|
+
const pages = await listPages({render: false}).catch(() => []);
|
|
129
129
|
const cutoff = Date.now() - STALE_PAGE_DAYS * 24 * 60 * 60 * 1000;
|
|
130
130
|
const stale = pages.filter(p => {
|
|
131
131
|
const updated = p.updatedAt ? new Date(p.updatedAt).getTime() : 0;
|
|
@@ -141,7 +141,7 @@ async function checkStalePages() {
|
|
|
141
141
|
}
|
|
142
142
|
|
|
143
143
|
async function checkMissingMeta() {
|
|
144
|
-
const pages = await listPages().catch(() => []);
|
|
144
|
+
const pages = await listPages({render: false}).catch(() => []);
|
|
145
145
|
const missing = pages.filter(p => !p.description || String(p.description).trim() === '');
|
|
146
146
|
return {
|
|
147
147
|
id: 'missing-meta',
|
|
@@ -96,6 +96,34 @@ export function htmlHash(html) {
|
|
|
96
96
|
return createHash('sha1').update(html).digest('base64');
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
// A page served from the response cache is the SAME string on every hit, and
|
|
100
|
+
// hashing it (sha1 over the whole document) was half of what a hit cost. The
|
|
101
|
+
// hash is kept per cache key beside the string it was made from, and used only
|
|
102
|
+
// while the caller hands in that very string - a page rendered again is a new
|
|
103
|
+
// string, fails the comparison (identity first, so a hit compares nothing) and
|
|
104
|
+
// is hashed afresh. So a kept hash can never describe other HTML. Dropped
|
|
105
|
+
// whenever anything leaves the response cache, so it does not hold a page the
|
|
106
|
+
// cache has let go.
|
|
107
|
+
const HASH_MEMO_MAX = 2000;
|
|
108
|
+
/** @type {Map<string, {html: string, hash: string}>} */
|
|
109
|
+
const _hashMemo = new Map();
|
|
110
|
+
cache.onInvalidate(() => _hashMemo.clear());
|
|
111
|
+
|
|
112
|
+
function memoisedHash(key, html) {
|
|
113
|
+
const kept = _hashMemo.get(key);
|
|
114
|
+
if (kept && kept.html === html) return kept.hash;
|
|
115
|
+
const hash = htmlHash(html);
|
|
116
|
+
_hashMemo.delete(key);
|
|
117
|
+
_hashMemo.set(key, {html, hash});
|
|
118
|
+
while (_hashMemo.size > HASH_MEMO_MAX) _hashMemo.delete(_hashMemo.keys().next().value);
|
|
119
|
+
return hash;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** For tests: how many hashes are kept. */
|
|
123
|
+
export function _hashMemoSize() {
|
|
124
|
+
return _hashMemo.size;
|
|
125
|
+
}
|
|
126
|
+
|
|
99
127
|
/**
|
|
100
128
|
* Send a rendered page: cache-control, an ETag, a 304 when the browser
|
|
101
129
|
* already has it, and - for a response that came from the response cache -
|
|
@@ -114,11 +142,12 @@ export function htmlHash(html) {
|
|
|
114
142
|
* @param {import('fastify').FastifyRequest} request
|
|
115
143
|
* @param {import('fastify').FastifyReply} reply
|
|
116
144
|
* @param {string} html
|
|
117
|
-
* @param {{cached?: boolean}} [opts] - cached: the HTML came from the response cache, so its
|
|
118
|
-
* compressed form may be kept too
|
|
145
|
+
* @param {{cached?: boolean, key?: string}} [opts] - cached: the HTML came from the response cache, so its
|
|
146
|
+
* compressed form may be kept too; key: the response-cache key it came from, so its hash is
|
|
147
|
+
* kept as well (only with `cached`)
|
|
119
148
|
*/
|
|
120
|
-
export async function sendHtml(request, reply, html, {cached = false} = {}) {
|
|
121
|
-
const hash = htmlHash(html);
|
|
149
|
+
export async function sendHtml(request, reply, html, {cached = false, key = ''} = {}) {
|
|
150
|
+
const hash = cached && key && cache.isEnabled() ? memoisedHash(key, html) : htmlHash(html);
|
|
122
151
|
const encoding = cached && Buffer.byteLength(html) >= COMPRESS_THRESHOLD
|
|
123
152
|
? pickEncoding(request.headers['accept-encoding'])
|
|
124
153
|
: null;
|
|
@@ -173,7 +173,7 @@ function toolTagFor(kind, raw, names) {
|
|
|
173
173
|
|
|
174
174
|
/** The items of a kind as `{id, tags, raw}`, through the same listers the routes use. */
|
|
175
175
|
async function items(kind) {
|
|
176
|
-
if (kind === 'pages') return (await listPages()).map(p => ({id: p.urlPath, tags: cleanTagList(p.adminTags), raw: p}));
|
|
176
|
+
if (kind === 'pages') return (await listPages({render: false})).map(p => ({id: p.urlPath, tags: cleanTagList(p.adminTags), raw: p}));
|
|
177
177
|
if (kind === 'collections') {
|
|
178
178
|
const map = readRegistry().collectionTags;
|
|
179
179
|
return (await listCollections()).map(c => ({id: c.slug, tags: cleanTagList(map[c.slug]), raw: c}));
|
|
@@ -2071,6 +2071,32 @@ export function unindentShortTags(markdown) {
|
|
|
2071
2071
|
return lines.map((l, i) => (layout[i].cut ? l.slice(layout[i].cut) : l)).join('\n');
|
|
2072
2072
|
}
|
|
2073
2073
|
|
|
2074
|
+
const SCRUB_PLACEHOLDER = /\x02SC(\d+):(\d+)\x03/g;
|
|
2075
|
+
const SHIELD_PLACEHOLDER = /DMSCT(\d+)x(\d+)x/g;
|
|
2076
|
+
|
|
2077
|
+
/**
|
|
2078
|
+
* The two patterns a shortcode name is matched with - self-closing and wrapping -
|
|
2079
|
+
* built once per name. `bounded`: the name must end there ([map] is not [map-guess]).
|
|
2080
|
+
*/
|
|
2081
|
+
const _shortcodePatterns = new Map();
|
|
2082
|
+
function shortcodePatterns(name, bounded) {
|
|
2083
|
+
const key = `${bounded ? 'b' : 'u'}:${name}`;
|
|
2084
|
+
let kept = _shortcodePatterns.get(key);
|
|
2085
|
+
if (!kept) {
|
|
2086
|
+
const end = bounded ? '(?=[\\s/\\]])' : '';
|
|
2087
|
+
kept = {
|
|
2088
|
+
selfClosing: new RegExp(`\\[${name}${end}([^\\]]*)\\s*\\/\\]`, 'gi'),
|
|
2089
|
+
wrapping: new RegExp(`\\[${name}${end}([^\\]]*)\\]([\\s\\S]*?)\\[\\/${name}\\]`, 'gi'),
|
|
2090
|
+
// What must be in the text (lower-cased) for either to match at all.
|
|
2091
|
+
needle: `[${String(name).toLowerCase()}`,
|
|
2092
|
+
// A name with pattern characters in it cannot be looked for as plain text.
|
|
2093
|
+
plain: /^[\w-]+$/.test(String(name))
|
|
2094
|
+
};
|
|
2095
|
+
_shortcodePatterns.set(key, kept);
|
|
2096
|
+
}
|
|
2097
|
+
return kept;
|
|
2098
|
+
}
|
|
2099
|
+
|
|
2074
2100
|
export function scrubCodeRegions(markdown, preserveIndented = false) {
|
|
2075
2101
|
const callId = ++scrubCallId;
|
|
2076
2102
|
const store = [];
|
|
@@ -2095,7 +2121,12 @@ export function scrubCodeRegions(markdown, preserveIndented = false) {
|
|
|
2095
2121
|
// A stored region can hold an earlier placeholder of this same call - an indented
|
|
2096
2122
|
// line (preserveIndented) holding an inline <code> - so what comes back is
|
|
2097
2123
|
// restored too. Restoring it once left "SC1:0" on the page.
|
|
2098
|
-
|
|
2124
|
+
// One shared pattern, the call checked in the callback: a pattern built per call
|
|
2125
|
+
// (its id in the source) is compiled afresh every time and never optimised, and
|
|
2126
|
+
// this runs many times for every page drawn. Another call's placeholders are left alone.
|
|
2127
|
+
const restore = (s) => (store.length
|
|
2128
|
+
? s.replace(SCRUB_PLACEHOLDER, (m, id, i) => (Number(id) === callId ? restore(store[Number(i)]) : m))
|
|
2129
|
+
: s);
|
|
2099
2130
|
return {scrubbed, restore};
|
|
2100
2131
|
}
|
|
2101
2132
|
|
|
@@ -2119,7 +2150,7 @@ function markedBody(md) {
|
|
|
2119
2150
|
const {scrubbed, restore} = scrubCodeRegions(text);
|
|
2120
2151
|
const shielded = restore(scrubbed.replace(ATTR_TAG, (tag) => { held.push(tag); return `DMSCT${id}x${held.length - 1}x`; }));
|
|
2121
2152
|
if (!held.length) return marked.parse(text);
|
|
2122
|
-
return marked.parse(shielded).replace(
|
|
2153
|
+
return marked.parse(shielded).replace(SHIELD_PLACEHOLDER, (m, call, i) => (Number(call) === id ? held[Number(i)] : m));
|
|
2123
2154
|
}
|
|
2124
2155
|
|
|
2125
2156
|
/**
|
|
@@ -2158,20 +2189,19 @@ async function processPluginShortcodes(markdown, user = null) {
|
|
|
2158
2189
|
return res + input.slice(lastEnd);
|
|
2159
2190
|
}
|
|
2160
2191
|
|
|
2192
|
+
// Most pages use none of most plugins' shortcodes: a plain-text look for "[name"
|
|
2193
|
+
// first, so a page pays two pattern scans only for the names it holds.
|
|
2194
|
+
let lower = result.toLowerCase();
|
|
2161
2195
|
for (const {name, handler} of processors) {
|
|
2196
|
+
// The name must end there: [map] is not [map-guess].
|
|
2197
|
+
const patterns = shortcodePatterns(name, true);
|
|
2198
|
+
if (patterns.plain && !lower.includes(patterns.needle)) continue;
|
|
2199
|
+
const before = result;
|
|
2162
2200
|
// Self-closing: [name attrs /]
|
|
2163
|
-
result = await replaceAsync(
|
|
2164
|
-
result,
|
|
2165
|
-
// The name must end there: [map] is not [map-guess].
|
|
2166
|
-
new RegExp(`\\[${name}(?=[\\s/\\]])([^\\]]*)\\s*\\/\\]`, 'gi'),
|
|
2167
|
-
(_, attrStr) => handler(attrStr, null, context)
|
|
2168
|
-
);
|
|
2201
|
+
result = await replaceAsync(result, patterns.selfClosing, (_, attrStr) => handler(attrStr, null, context));
|
|
2169
2202
|
// Wrapping: [name attrs]...[/name]
|
|
2170
|
-
result = await replaceAsync(
|
|
2171
|
-
|
|
2172
|
-
new RegExp(`\\[${name}(?=[\\s/\\]])([^\\]]*)\\]([\\s\\S]*?)\\[\\/${name}\\]`, 'gi'),
|
|
2173
|
-
(_, attrStr, body) => handler(attrStr, body, context)
|
|
2174
|
-
);
|
|
2203
|
+
result = await replaceAsync(result, patterns.wrapping, (_, attrStr, body) => handler(attrStr, body, context));
|
|
2204
|
+
if (result !== before) lower = result.toLowerCase();
|
|
2175
2205
|
}
|
|
2176
2206
|
return restore(result);
|
|
2177
2207
|
}
|
|
@@ -2189,15 +2219,15 @@ function processEffectsBlocks(markdown) {
|
|
|
2189
2219
|
const {scrubbed, restore} = scrubCodeRegions(markdown);
|
|
2190
2220
|
let result = scrubbed;
|
|
2191
2221
|
|
|
2222
|
+
// As with plugin shortcodes: look for "[name" as plain text before scanning with the patterns.
|
|
2223
|
+
let lower = result.toLowerCase();
|
|
2192
2224
|
function apply(name, handler) {
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
);
|
|
2197
|
-
result = result.replace(
|
|
2198
|
-
|
|
2199
|
-
(_, attrStr, body) => handler(attrStr, body)
|
|
2200
|
-
);
|
|
2225
|
+
const patterns = shortcodePatterns(name, false);
|
|
2226
|
+
if (patterns.plain && !lower.includes(patterns.needle)) return;
|
|
2227
|
+
const before = result;
|
|
2228
|
+
result = result.replace(patterns.selfClosing, (_, attrStr) => handler(attrStr, null));
|
|
2229
|
+
result = result.replace(patterns.wrapping, (_, attrStr, body) => handler(attrStr, body));
|
|
2230
|
+
if (result !== before) lower = result.toLowerCase();
|
|
2201
2231
|
}
|
|
2202
2232
|
|
|
2203
2233
|
apply('counter', (attrStr) => {
|
|
@@ -2230,6 +2260,7 @@ function processEffectsBlocks(markdown) {
|
|
|
2230
2260
|
// A single inline character also accepts the concise, unpaired form.
|
|
2231
2261
|
result = result.replace(/\[celebrate([^\]]*)\]/gi, (match, attrs) =>
|
|
2232
2262
|
parseShortcodeAttrs(attrs).character ? renderCelebration(attrs) : match);
|
|
2263
|
+
lower = result.toLowerCase();
|
|
2233
2264
|
|
|
2234
2265
|
// Named celebration shortcodes - sugar for [celebrate theme="..."] with override support
|
|
2235
2266
|
['christmas', 'halloween', 'st-patricks', 'valentines', 'guy-fawkes',
|
|
@@ -5406,7 +5437,10 @@ export function sanitizePageHtml(html) {
|
|
|
5406
5437
|
*/
|
|
5407
5438
|
export async function parseMarkdown(raw, opts = {}) {
|
|
5408
5439
|
opts = {...opts, [SLOT_BUDGET]:opts[SLOT_BUDGET] || {remaining:500}};
|
|
5409
|
-
|
|
5440
|
+
// Options passed on purpose: without them gray-matter keeps every string it is
|
|
5441
|
+
// given for the life of the process - each editor preview, each version of each
|
|
5442
|
+
// page - and hands every caller the same `data` object.
|
|
5443
|
+
const {data, content: written} = matter(raw, {});
|
|
5410
5444
|
const content = unindentShortTags(written);
|
|
5411
5445
|
|
|
5412
5446
|
// Pipeline (earlier stages are already HTML when later ones run; keep in step
|
|
@@ -78,7 +78,7 @@ export function titleFromSlug(slug) {
|
|
|
78
78
|
*/
|
|
79
79
|
export function pageMeta(slug, source) {
|
|
80
80
|
let data = {};
|
|
81
|
-
try { data = matter(source).data || {}; } catch { data = {}; }
|
|
81
|
+
try { data = matter(source, {}).data || {}; } catch { data = {}; }
|
|
82
82
|
const title = typeof data.title === 'string' && data.title.trim() ? data.title.trim() : titleFromSlug(slug);
|
|
83
83
|
const order = Number.isFinite(Number(data.order)) && data.order !== null && data.order !== ''
|
|
84
84
|
? Number(data.order)
|
|
@@ -123,7 +123,7 @@ const SANITIZE = {
|
|
|
123
123
|
*/
|
|
124
124
|
export function renderGuide(source) {
|
|
125
125
|
let body = source;
|
|
126
|
-
try { body = matter(source).content; } catch { /* no frontmatter */ }
|
|
126
|
+
try { body = matter(source, {}).content; } catch { /* no frontmatter */ }
|
|
127
127
|
return sanitizeHtml(marked.parse(body, {gfm: true, async: false}), SANITIZE);
|
|
128
128
|
}
|
|
129
129
|
|