domma-cms 1.61.0 → 1.61.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/CLAUDE.md +1 -1
  3. package/docs/DommaFeatures.md +3 -1
  4. package/docs/cache.md +24 -0
  5. package/docs/configuration.md +1 -1
  6. package/package.json +1 -1
  7. package/plugins/_lib/admin/mail/body-menu.js +1 -1
  8. package/plugins/_lib/admin/mail/cms-host.js +1 -1
  9. package/plugins/_lib/admin/mail/compose-window.js +1 -1
  10. package/plugins/_lib/admin/mail/contacts.js +1 -1
  11. package/plugins/_lib/admin/mail/diagnostics-section.js +1 -1
  12. package/plugins/_lib/admin/mail/drag-drop.js +1 -1
  13. package/plugins/_lib/admin/mail/folder-tree.js +1 -1
  14. package/plugins/_lib/admin/mail/font-controls.js +1 -1
  15. package/plugins/_lib/admin/mail/fonts.js +1 -1
  16. package/plugins/_lib/admin/mail/host-context.js +1 -1
  17. package/plugins/_lib/admin/mail/identity.js +1 -1
  18. package/plugins/_lib/admin/mail/image-senders-section.js +1 -1
  19. package/plugins/_lib/admin/mail/keyboard.js +1 -1
  20. package/plugins/_lib/admin/mail/mail.html +1 -1
  21. package/plugins/_lib/admin/mail/message-actions.js +1 -1
  22. package/plugins/_lib/admin/mail/oauth-panel.js +1 -1
  23. package/plugins/_lib/admin/mail/panes.js +1 -1
  24. package/plugins/_lib/admin/mail/reader-kit.js +1 -1
  25. package/plugins/_lib/admin/mail/reader-runtime.js +27 -1
  26. package/plugins/_lib/admin/mail/reader-view.js +1 -1
  27. package/plugins/_lib/admin/mail/resizable.js +1 -1
  28. package/plugins/_lib/admin/mail/restore-spam.js +1 -1
  29. package/plugins/_lib/admin/mail/rules.js +1 -1
  30. package/plugins/_lib/admin/mail/scheduling.js +1 -1
  31. package/plugins/_lib/admin/mail/section-kit.js +1 -1
  32. package/plugins/_lib/admin/mail/security-note.js +1 -1
  33. package/plugins/_lib/admin/mail/settings-section.js +1 -1
  34. package/plugins/_lib/admin/mail/templates.js +1 -1
  35. package/plugins/_lib/admin/mail/threads.js +1 -1
  36. package/plugins/_lib/admin/mail/vacation.js +1 -1
  37. package/plugins/_lib/admin/mail/when-closed.js +1 -1
  38. package/plugins/_lib/admin/ui/dates.js +1 -1
  39. package/plugins/_lib/admin/ui/resizable.js +1 -1
  40. package/plugins/_lib/domma-mail.lock.json +80 -80
  41. package/plugins/_lib/mail/accountFileStore.js +1 -1
  42. package/plugins/_lib/mail/accountRepository.js +1 -1
  43. package/plugins/_lib/mail/accountRoutes.js +1 -1
  44. package/plugins/_lib/mail/bodyTokenProvider.js +1 -1
  45. package/plugins/_lib/mail/bodyTokens.js +1 -1
  46. package/plugins/_lib/mail/compose.js +1 -1
  47. package/plugins/_lib/mail/defaults.js +1 -1
  48. package/plugins/_lib/mail/envelope.js +1 -1
  49. package/plugins/_lib/mail/folderActions.js +1 -1
  50. package/plugins/_lib/mail/folderRuntime.js +1 -1
  51. package/plugins/_lib/mail/folders.js +1 -1
  52. package/plugins/_lib/mail/imapPoolRuntime.js +1 -1
  53. package/plugins/_lib/mail/mbox.js +1 -1
  54. package/plugins/_lib/mail/message-action.js +1 -1
  55. package/plugins/_lib/mail/messageSecurity.js +1 -1
  56. package/plugins/_lib/mail/messageSecurityProvider.js +1 -1
  57. package/plugins/_lib/mail/networkAddresses.js +1 -1
  58. package/plugins/_lib/mail/networkPolicy.js +1 -1
  59. package/plugins/_lib/mail/oauthAccounts.js +1 -1
  60. package/plugins/_lib/mail/oauthRouteRuntime.js +1 -1
  61. package/plugins/_lib/mail/oauthRuntime.js +1 -1
  62. package/plugins/_lib/mail/pollSchedule.js +1 -1
  63. package/plugins/_lib/mail/poller.js +1 -1
  64. package/plugins/_lib/mail/pollerRuntime.js +1 -1
  65. package/plugins/_lib/mail/priority.js +1 -1
  66. package/plugins/_lib/mail/publicMailbox.js +1 -1
  67. package/plugins/_lib/mail/readRuntime.js +1 -1
  68. package/plugins/_lib/mail/render.js +1 -1
  69. package/plugins/_lib/mail/ruleRunner.js +1 -1
  70. package/plugins/_lib/mail/schedulerRuntime.js +1 -1
  71. package/plugins/_lib/mail/secretProvider.js +1 -1
  72. package/plugins/_lib/mail/secretbox.js +1 -1
  73. package/plugins/_lib/mail/senderAuth.js +1 -1
  74. package/plugins/_lib/mail/sieve.js +1 -1
  75. package/plugins/_lib/mail/smtpService.js +1 -1
  76. package/plugins/_lib/mail/snoozeRuntime.js +1 -1
  77. package/plugins/_lib/mail/store.js +1 -1
  78. package/plugins/_lib/mail/subscriptions.js +1 -1
  79. package/plugins/_lib/mail/syncPlan.js +1 -1
  80. package/plugins/_lib/mail/syncRuntime.js +1 -1
  81. package/plugins/_lib/mail/syncSelection.js +1 -1
  82. package/plugins/_lib/mail/unreadBadge.js +1 -1
  83. package/plugins/_lib/mail/unsubscribe.js +1 -1
  84. package/plugins/_lib/mail/vacationRuntime.js +1 -1
  85. package/plugins/_lib/mail/write.js +1 -1
  86. package/plugins/docs/CHANGELOG.md +5 -0
  87. package/plugins/docs/CLAUDE.md +30 -0
  88. package/plugins/docs/admin/views/docs.js +2 -1
  89. package/plugins/docs/admin/views/editor.js +5 -3
  90. package/plugins/docs/docs/guide.md +11 -0
  91. package/plugins/docs/lib/store.js +46 -2
  92. package/plugins/docs/plugin.js +70 -28
  93. package/plugins/docs/plugin.json +14 -1
  94. package/plugins/docs/tests/restricted.test.js +240 -0
  95. package/plugins/free-tier.lock.json +15 -13
  96. package/plugins/mail-reader/CLAUDE.md +6 -0
  97. package/plugins/mail-reader/plugin.js +9 -0
  98. package/plugins/mail-reader/plugin.json +1 -1
  99. package/public/js/effects.js +1 -1
  100. package/public/js/site.js +1 -1
  101. package/server/config.js +37 -0
  102. package/server/routes/api/chats.js +3 -1
  103. package/server/routes/api/components.js +2 -1
  104. package/server/routes/api/content-api.js +1 -1
  105. package/server/routes/api/content-public.js +2 -2
  106. package/server/routes/api/content-writes.js +1 -1
  107. package/server/routes/api/effects.js +4 -9
  108. package/server/routes/api/layouts.js +1 -1
  109. package/server/routes/api/pages.js +6 -5
  110. package/server/routes/api/search.js +3 -3
  111. package/server/routes/api/sidebar-counts.js +2 -0
  112. package/server/routes/api/theme.js +2 -1
  113. package/server/routes/api/views.js +2 -1
  114. package/server/routes/public.js +4 -4
  115. package/server/server.js +12 -6
  116. package/server/services/cache/adminChange.js +28 -0
  117. package/server/services/content.js +73 -10
  118. package/server/services/effectsSettings.js +25 -0
  119. package/server/services/health.js +2 -2
  120. package/server/services/htmlResponse.js +33 -4
  121. package/server/services/listTags.js +1 -1
  122. package/server/services/markdown.js +56 -22
  123. package/server/services/pluginGuides.js +2 -2
  124. package/server/services/plugins.js +25 -2
  125. package/server/services/projects.js +39 -4
  126. package/server/services/publicAssets.js +488 -0
  127. package/server/services/renderer.js +10 -8
  128. package/server/services/seo/index.js +3 -3
  129. package/server/services/sitemap.js +1 -1
  130. package/server/services/versions.js +2 -2
  131. package/server/services/wpImport/index.js +2 -2
  132. package/server/templates/page.html +3 -2
@@ -110,7 +110,8 @@ export async function pagesRoutes(fastify) {
110
110
  const canDelete = {preHandler: [authenticate, requirePermission('pages', 'delete')]};
111
111
 
112
112
  // Render markdown preview (shortcodes + sanitize, no frontmatter)
113
- fastify.post('/pages/preview', canRead, async (request, reply) => {
113
+ // The two previews render and return; the editor calls them as you type. Not a change (cache/adminChange.js).
114
+ fastify.post('/pages/preview', {...canRead, config: {keepsCache: true}}, async (request, reply) => {
114
115
  const {markdown} = request.body;
115
116
  if (typeof markdown !== 'string') return reply.status(400).send({error: 'markdown must be a string'});
116
117
  const {html} = await parseMarkdown(`---\ntitle: preview\n---\n${markdown}`, {user: request.user || null});
@@ -125,7 +126,7 @@ export async function pagesRoutes(fastify) {
125
126
  // theme, menus, dconfig and custom CSS are all exactly what shipping
126
127
  // would produce. No draft banner: the editor is already telling the user
127
128
  // what they are looking at.
128
- fastify.post('/pages/preview/full', canRead, async (request, reply) => {
129
+ fastify.post('/pages/preview/full', {...canRead, config: {keepsCache: true}}, async (request, reply) => {
129
130
  const {frontmatter = {}, body = '', urlPath = '/'} = request.body || {};
130
131
  if (typeof body !== 'string') return reply.status(400).send({error: 'body must be a string'});
131
132
  if (typeof urlPath !== 'string' || !urlPath.startsWith('/')) {
@@ -199,7 +200,7 @@ export async function pagesRoutes(fastify) {
199
200
 
200
201
  // Aggregate unique tags from all pages - must be registered before the wildcard /pages/* route
201
202
  fastify.get('/pages/tags', canRead, async (request, reply) => {
202
- const pages = (await listPages()).filter(p => inScope(request.user, p));
203
+ const pages = (await listPages({render: false})).filter(p => inScope(request.user, p));
203
204
  const tagSet = new Set();
204
205
  pages.forEach(p => {
205
206
  if (Array.isArray(p.tags)) p.tags.forEach(t => { if (t) tagSet.add(t); });
@@ -209,7 +210,7 @@ export async function pagesRoutes(fastify) {
209
210
 
210
211
  // The sidebar badge. Static, so it is matched before the /pages/* wildcard below.
211
212
  fastify.get('/pages/_count', canRead, async (request) =>
212
- pagesBadge((await listPages()).filter(p => inScope(request.user, p))));
213
+ pagesBadge((await listPages({render: false})).filter(p => inScope(request.user, p))));
213
214
 
214
215
  // Domma's option types for the editor (admin/js/lib/dconfig-check.js). Static, before /pages/*.
215
216
  fastify.get('/pages/_domma-catalogue', canRead, async (request, reply) => {
@@ -220,7 +221,7 @@ export async function pagesRoutes(fastify) {
220
221
 
221
222
  // List all pages
222
223
  fastify.get('/pages', canRead, async (request, reply) => {
223
- const pages = (await listPages()).filter(p => inScope(request.user, p));
224
+ const pages = (await listPages({render: false})).filter(p => inScope(request.user, p));
224
225
  const counts = await Promise.all(pages.map(p => getVersionCount(p.urlPath)));
225
226
  return pages.map((p, i) => ({
226
227
  urlPath: p.urlPath,
@@ -57,7 +57,7 @@ export async function searchRoutes(fastify, opts = {}) {
57
57
  // no-store (the /api onSend hook) and never enter the response cache,
58
58
  // so nothing gated can be replayed to another viewer.
59
59
  const viewer = await resolveViewer(request);
60
- const pages = await listPages();
60
+ const pages = await listPages({render: false});
61
61
  const results = searchPages(pages, q, {
62
62
  maxResults: settings.maxResults,
63
63
  viewer,
@@ -81,7 +81,7 @@ export async function searchRoutes(fastify, opts = {}) {
81
81
  const settings = getSearchSettings();
82
82
  const asked = parseInt(request.query.max, 10);
83
83
  const maxResults = Number.isFinite(asked) ? Math.min(50, Math.max(1, asked)) : settings.maxResults;
84
- const pages = await listPages();
84
+ const pages = await listPages({render: false});
85
85
  const exclusions = await searchExclusions(pages);
86
86
  const results = searchPages(pages, q, {
87
87
  maxResults,
@@ -92,7 +92,7 @@ export async function searchRoutes(fastify, opts = {}) {
92
92
  });
93
93
 
94
94
  fastify.get('/search/_count', canReadBadge, async () => {
95
- const pages = await listPages();
95
+ const pages = await listPages({render: false});
96
96
  return searchBadge(getSearchSettings(), pages, await searchExclusions(pages));
97
97
  });
98
98
  }
@@ -8,6 +8,8 @@ export async function sidebarCountRoutes(fastify, opts = {}) {
8
8
  const isItemAllowed = opts.isItemAllowed || (() => true);
9
9
  const timeoutMs = opts.countTimeoutMs || 5000;
10
10
  fastify.post('/sidebar/counts', {
11
+ // A read that happens to be a POST, polled by every open admin: not a change (cache/adminChange.js).
12
+ config: {keepsCache: true},
11
13
  preHandler: [authenticate], bodyLimit: 32 * 1024,
12
14
  schema: {body: {type: 'object', additionalProperties: false, properties: {
13
15
  urls: {type: 'array', maxItems: 256, uniqueItems: true, items: {type: 'string', maxLength: 512}},
@@ -142,7 +142,8 @@ export async function themeRoutes(fastify, opts = {}) {
142
142
  }
143
143
  });
144
144
 
145
- fastify.post('/theme/compile', canRead, async (request, reply) => {
145
+ // Compiles what it is sent and returns it: not a change (cache/adminChange.js).
146
+ fastify.post('/theme/compile', {...canRead, config: {keepsCache: true}}, async (request, reply) => {
146
147
  const candidate = request.body || {};
147
148
  const errors = validateThemeConfig(candidate, {remembered: rememberedIds(await loadThemeConfig())});
148
149
  if (errors.length) return reply.code(400).send({error: errors.join('; '), validation: errors});
@@ -181,7 +181,8 @@ export async function viewsRoutes(fastify) {
181
181
  // user may see (project scope), and it runs as an admin preview - no
182
182
  // row-level filter, the same as executing a saved view from the admin.
183
183
  // `_` is outside the slug alphabet, so no view can shadow the route.
184
- fastify.post('/views/_preview', canUpdate, async (request, reply) => {
184
+ // Runs an unsaved view and returns its rows: not a change (cache/adminChange.js).
185
+ fastify.post('/views/_preview', {...canUpdate, config: {keepsCache: true}}, async (request, reply) => {
185
186
  const {source, connection = 'default', stages = [], limit} = request.body || {};
186
187
  if (typeof source !== 'string' || !source) return reply.status(400).send({error: 'Choose a source collection'});
187
188
  if (!Array.isArray(stages)) return reply.status(400).send({error: 'stages must be an array'});
@@ -17,7 +17,7 @@ import {validSessionCookie, newSessionCookie} from '../services/shortcodeExpiry.
17
17
  import {touchPreviewLink, verifyPreviewToken} from '../services/previewLinks.js';
18
18
  import {viewerRoleKey} from '../services/userRoles.js';
19
19
  import {hooks} from '../services/hooks.js';
20
- import {getConfig} from '../config.js';
20
+ import {readConfig} from '../config.js';
21
21
  import * as cache from '../services/cache/index.js';
22
22
 
23
23
  /**
@@ -34,7 +34,7 @@ import * as cache from '../services/cache/index.js';
34
34
  * @returns {string} e.g. 'https://example.com' (no trailing slash)
35
35
  */
36
36
  export function getBaseUrl(request) {
37
- const override = (getConfig('site') || {}).baseUrl;
37
+ const override = (readConfig('site') || {}).baseUrl;
38
38
  if (override) return override.toString().trim().replace(/\/+$/, '');
39
39
  const host = request.host || request.headers.host || request.hostname;
40
40
  return `${request.protocol}://${host}`;
@@ -283,7 +283,7 @@ export async function publicRoutes(fastify) {
283
283
  const pageMtime = await getPageMtime(page.urlPath || urlPath);
284
284
  const cacheKey = `page:${urlPath}:m${pageMtime}:role${viewerRoleKey(userObj)}:o${baseUrl}`;
285
285
  // The footer's blocks (site.json footer.blocks) are on every page: editing one must reach the cached pages.
286
- const cacheTags = [`page:${urlPath}`, ...(page.cacheTags || []), 'nav', 'site', ...footerBlockCacheTags(getConfig('site'))];
286
+ const cacheTags = [`page:${urlPath}`, ...(page.cacheTags || []), 'nav', 'site', ...footerBlockCacheTags(readConfig('site'))];
287
287
  // x-dm-cache says which way this response went - hit, miss, off (the
288
288
  // cache is disabled) or bypass (a draft, an expiring or a per-viewer
289
289
  // page, never cached) - so a slow page can be read from the browser.
@@ -294,7 +294,7 @@ export async function publicRoutes(fastify) {
294
294
  reply.header('x-dm-cache', status);
295
295
  hooks.emit('content:pageViewed', {urlPath, title: page.title || ''});
296
296
  // cached: this HTML is shared, so its compressed body is kept too (htmlResponse.js).
297
- return sendHtml(request, reply, html, {cached: true});
297
+ return sendHtml(request, reply, html, {cached: true, key: cacheKey});
298
298
  });
299
299
  }
300
300
 
package/server/server.js CHANGED
@@ -19,7 +19,7 @@ import path from 'path';
19
19
  import fs from 'fs/promises';
20
20
  import {fileURLToPath} from 'url';
21
21
  import {createRequire} from 'module';
22
- import {config, getConfig} from './config.js';
22
+ import {config, getConfig, readConfig} from './config.js';
23
23
  import {getLoadedPlugins, getPluginSettings, recordPluginLoadFailure, registerPlugins} from './services/plugins.js';
24
24
  import {notifierFor} from './services/notify.js';
25
25
  import {grantAdminToolDefaults, grantPermissionOnce, load as loadRoles, seed as seedRoles} from './services/roles.js';
@@ -34,6 +34,7 @@ import {parseTrustProxy} from './services/network.js';
34
34
  import {requestSerializer} from './services/request-log.js';
35
35
  import {rateLimitOptions} from './services/rate-limit.js';
36
36
  import {registerAdminAssets} from './services/adminAssets.js';
37
+ import {registerPublicAssets} from './services/publicAssets.js';
37
38
  import {isActiveType, servedType} from './services/mediaTypes.js';
38
39
  import {connectSources, frameSources, imageSources} from './services/embedCsp.js';
39
40
  import {cspSources as gaSources} from './services/googleAnalytics.js';
@@ -131,7 +132,7 @@ await app.register(helmet, {
131
132
  scriptSrc: ["'self'", "'unsafe-inline'", "'wasm-unsafe-eval'", 'https://cdn.jsdelivr.net', gaSources('script')],
132
133
  styleSrc: ["'self'", "'unsafe-inline'", 'https://cdn.jsdelivr.net', 'https://fonts.googleapis.com'],
133
134
  // config/server.json csp.imgHosts: other sites a page may show images from (services/embedCsp.js).
134
- imgSrc: [() => imageSources(getConfig('server').csp?.imgHosts).join(' '), gaSources('img')],
135
+ imgSrc: [() => imageSources(readConfig('server').csp?.imgHosts).join(' '), gaSources('img')],
135
136
  fontSrc: ["'self'", 'data:', 'https://fonts.gstatic.com'],
136
137
  // config/server.json csp.connectHosts: other sites a page may read from (services/embedCsp.js).
137
138
  connectSrc: [...connectSources(serverConfig.csp?.connectHosts), gaSources('connect')],
@@ -399,11 +400,11 @@ try {
399
400
  * tag. Runs on onResponse, after the reply has gone, so it costs the caller
400
401
  * nothing - and a memory-driver clear is two Map resets.
401
402
  */
402
- const CACHE_CLEARING_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
403
+ // A route that changes nothing a page could show opts out with `config: {keepsCache: true}`
404
+ // (services/cache/adminChange.js) - the admin's 30-second chat check-in was emptying the cache.
405
+ const {isCacheClearingChange} = await import('./services/cache/adminChange.js');
403
406
  app.addHook('onResponse', async (request, reply) => {
404
- if (!CACHE_CLEARING_METHODS.has(request.method)) return;
405
- if (reply.statusCode >= 400) return;
406
- if (!request.user) return;
407
+ if (!isCacheClearingChange(request, reply)) return;
407
408
  try {
408
409
  await cache.clear();
409
410
  } catch (err) {
@@ -442,6 +443,10 @@ await app.register(staticPlugin, {
442
443
  }
443
444
  });
444
445
 
446
+ // Fingerprinted public assets are cached for good, and the site's custom CSS is
447
+ // served as a file - services/publicAssets.js.
448
+ const publicAssets = registerPublicAssets(app, {root: ROOT, dommaDist: DOMMA_DIST, pluginsDir});
449
+
445
450
  // Serve plugin admin/ and public/ subdirs only - block plugin.js, config.js, data/, etc.
446
451
  await app.register(async function pluginStaticScope(instance) {
447
452
  instance.addHook('onRequest', (request, reply, done) => {
@@ -792,6 +797,7 @@ await registerPlugins(app);
792
797
  }
793
798
  // After the plugins' onRequest hooks - see serve() in services/adminAssets.js.
794
799
  adminAssets.serve();
800
+ publicAssets.serve();
795
801
 
796
802
  // ---------------------------------------------------------------------------
797
803
  // Public Plugin Routes (root-level, before catch-all)
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Does this finished request count as an admin change that clears the
3
+ * response cache? (The safety net in server.js - see the comment there.)
4
+ *
5
+ * A signed-in, successful POST/PUT/PATCH/DELETE does - unless its route says
6
+ * it changes nothing a page could show: `config: {keepsCache: true}`. The
7
+ * admin's "available for chats" check-in is one, sent every 30 seconds from
8
+ * every admin screen; counted as a change, one signed-in person emptied the
9
+ * whole cache twice a minute and no page was ever served from it.
10
+ *
11
+ * Only for a route whose effect lives outside every rendered page. When in
12
+ * doubt leave it off: a cache cleared too often is slow, a cache not cleared
13
+ * is wrong.
14
+ */
15
+ const CLEARING_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
16
+
17
+ /**
18
+ * @param {{method: string, user?: object, routeOptions?: {config?: {keepsCache?: boolean}}}} request
19
+ * @param {{statusCode: number}} reply
20
+ * @returns {boolean}
21
+ */
22
+ export function isCacheClearingChange(request, reply) {
23
+ if (!CLEARING_METHODS.has(request.method)) return false;
24
+ if (reply.statusCode >= 400) return false;
25
+ if (!request.user) return false;
26
+ if (request.routeOptions?.config?.keepsCache === true) return false;
27
+ return true;
28
+ }
@@ -35,6 +35,9 @@ const MEDIA_DIR = config.content.mediaDir;
35
35
  // the hooks do not know about. Entries are cloned in and out, so a caller
36
36
  // that edits the object it was handed edits its own copy.
37
37
  const PAGE_MEMO_MAX = 500;
38
+ /** listPages() raises the limit to the number of pages, up to this. */
39
+ const PAGE_MEMO_CEILING = 5000;
40
+ let _pageMemoMax = PAGE_MEMO_MAX;
38
41
  const PAGE_MEMO_TTL_MS = 5 * 60 * 1000;
39
42
  /** @type {Map<string, {mtimeMs: number, size: number, at: number, page: object}>} */
40
43
  const _pageMemo = new Map();
@@ -43,11 +46,13 @@ cache.onInvalidate(() => _pageMemo.clear());
43
46
 
44
47
  /** For the cache status route and tests: how the memo is doing. */
45
48
  export function pageMemoStats() {
46
- return {size: _pageMemo.size, max: PAGE_MEMO_MAX, hits: _pageMemoStats.hits, misses: _pageMemoStats.misses};
49
+ return {size: _pageMemo.size, max: _pageMemoMax, hits: _pageMemoStats.hits, misses: _pageMemoStats.misses};
47
50
  }
48
51
 
49
52
  export function _resetPageMemoForTests() {
50
53
  _pageMemo.clear();
54
+ _summaryMemo.clear();
55
+ _pageMemoMax = PAGE_MEMO_MAX;
51
56
  _pageMemoStats.hits = 0;
52
57
  _pageMemoStats.misses = 0;
53
58
  }
@@ -69,7 +74,7 @@ async function readPageMemo(filePath) {
69
74
  } catch {
70
75
  return page; // something in it cannot be cloned: hand it over unmemoised
71
76
  }
72
- while (_pageMemo.size > PAGE_MEMO_MAX) _pageMemo.delete(_pageMemo.keys().next().value);
77
+ while (_pageMemo.size > _pageMemoMax) _pageMemo.delete(_pageMemo.keys().next().value);
73
78
  return page;
74
79
  }
75
80
 
@@ -77,21 +82,80 @@ async function readPageMemo(filePath) {
77
82
  // Pages
78
83
  // ---------------------------------------------------------------------------
79
84
 
85
+ // ---------------------------------------------------------------------------
86
+ // Page summaries (front matter + source, nothing rendered)
87
+ // ---------------------------------------------------------------------------
88
+ //
89
+ // A page's front matter and Markdown source depend on its file and nothing
90
+ // else, so they are kept per file and checked against the file's mtime and
91
+ // size on every read - no TTL, and correct with the response cache on or off.
92
+ // Emptied with the response cache all the same, so two saves inside one mtime
93
+ // tick cannot leave the first one's values behind. Cloned out, so a caller
94
+ // that edits a row edits its own copy.
95
+ /** @type {Map<string, {mtimeMs: number, size: number, page: object}>} */
96
+ const _summaryMemo = new Map();
97
+ cache.onInvalidate(() => _summaryMemo.clear());
98
+
99
+ async function readPageSummary(filePath) {
100
+ const st = await fs.stat(filePath);
101
+ const kept = _summaryMemo.get(filePath);
102
+ if (kept && kept.mtimeMs === st.mtimeMs && kept.size === st.size) return structuredClone(kept.page);
103
+ const raw = await fs.readFile(filePath, 'utf8');
104
+ // Options passed on purpose: without them gray-matter keeps every string it
105
+ // has ever parsed (and hands the same `data` object to every caller).
106
+ const {data, content} = matter(raw, {});
107
+ const page = {...data, urlPath: filePathToUrlPath(filePath), content};
108
+ try {
109
+ _summaryMemo.set(filePath, {mtimeMs: st.mtimeMs, size: st.size, page: structuredClone(page)});
110
+ } catch { /* something in the front matter cannot be cloned: read it each time */ }
111
+ return page;
112
+ }
113
+
80
114
  /**
81
115
  * Recursively list all .md files under PAGES_DIR.
82
- * Returns parsed page objects sorted by sortOrder then title.
116
+ * Returns page objects sorted by sortOrder then title.
117
+ *
118
+ * By default every page is parsed in full, as getPage() does - shortcodes,
119
+ * collections, sanitising - which is tens of milliseconds a page and seconds
120
+ * for a few hundred. A caller that reads only the front matter (title, status,
121
+ * layout, tags, seo, ...) or the Markdown source (`content`) asks for
122
+ * `{render: false}` and gets exactly those plus `urlPath` and
123
+ * `resolvedProject`: no `html`, `usedComponents`, `cacheTags`, `perViewer` or
124
+ * `hasExpiry`. Lists, badges, search and the sitemap all do.
125
+ *
126
+ * The rendered rows come through the parsed-page memo while the response
127
+ * cache is on, so a second listing does not parse every page again.
83
128
  *
129
+ * @param {{render?: boolean}} [opts]
84
130
  * @returns {Promise<object[]>}
85
131
  */
86
- export async function listPages() {
132
+ export async function listPages({render = true} = {}) {
87
133
  const files = await collectMdFiles(PAGES_DIR);
88
- const pages = await Promise.all(files.map(filePath => readPageFile(filePath)));
134
+ let pages;
135
+ if (!render) {
136
+ pages = await Promise.all(files.map(filePath => readPageSummary(filePath)));
137
+ // Forget pages that are gone, so the memo holds the site's pages and no more.
138
+ if (_summaryMemo.size > files.length) {
139
+ const live = new Set(files);
140
+ for (const key of _summaryMemo.keys()) if (!live.has(key)) _summaryMemo.delete(key);
141
+ }
142
+ } else if (cache.isEnabled()) {
143
+ // Every page must fit at once, or a listing evicts what it is about to read.
144
+ _pageMemoMax = Math.max(_pageMemoMax, Math.min(PAGE_MEMO_CEILING, files.length + 50));
145
+ pages = await Promise.all(files.map(filePath => readPageMemo(filePath)));
146
+ } else {
147
+ pages = await Promise.all(files.map(filePath => readPageFile(filePath)));
148
+ }
89
149
  pages.sort((a, b) => (a.sortOrder ?? 99) - (b.sortOrder ?? 99) || a.title.localeCompare(b.title));
90
- const {getProjectForPage} = await import('./projects.js');
150
+ // One read of the projects for the whole list, not one per page.
151
+ const {listProjects, projectForPage} = await import('./projects.js');
152
+ let projects = null;
91
153
  for (const page of pages) {
92
154
  const url = page.url || page.urlPath || '/';
93
155
  const explicit = (page.project === undefined) ? undefined : page.project;
94
- page.resolvedProject = await getProjectForPage(url, explicit);
156
+ if (typeof explicit === 'string' && explicit) { page.resolvedProject = explicit; continue; }
157
+ projects ??= await listProjects();
158
+ page.resolvedProject = projectForPage(projects, url, explicit);
95
159
  }
96
160
  return pages;
97
161
  }
@@ -327,9 +391,8 @@ export async function setPageFrontmatter(urlPath, patch) {
327
391
  throw err;
328
392
  }
329
393
  const raw = await fs.readFile(filePath, 'utf8');
330
- // gray-matter caches parses by input string and hands back the same
331
- // object: copy before changing it.
332
- const parsed = matter(raw);
394
+ // Options passed so gray-matter does not keep the string (see readPageSummary).
395
+ const parsed = matter(raw, {});
333
396
  const data = {...parsed.data};
334
397
  for (const [k, v] of Object.entries(patch || {})) {
335
398
  if (v === undefined) delete data[k];
@@ -0,0 +1,25 @@
1
+ /**
2
+ * The scroll-reveal defaults every public page's effects read
3
+ * (config/effects.json over the built-in defaults).
4
+ *
5
+ * Its own module so both the route that serves them and the renderer - which
6
+ * writes them into the page, saving every page view a request before its
7
+ * effects can start - read the same thing.
8
+ */
9
+ import {getConfig} from '../config.js';
10
+
11
+ export const EFFECTS_DEFAULTS = {respectMotion: true, defaultDuration: 600, defaultAnimation: 'fade', defaultThreshold: 0.1};
12
+
13
+ /** @returns {object} the defaults with config/effects.json over them */
14
+ export function loadEffectsSettings() {
15
+ try {
16
+ return {...EFFECTS_DEFAULTS, ...(getConfig('effects') || {})};
17
+ } catch {
18
+ return {...EFFECTS_DEFAULTS};
19
+ }
20
+ }
21
+
22
+ /** `window.__CMS_EFFECTS__ = {...};` for a page's inline script (public/js/effects.js reads it). */
23
+ export function effectsSettingsScript() {
24
+ return `window.__CMS_EFFECTS__ = ${JSON.stringify(loadEffectsSettings()).replace(/</g, '\\u003c')};`;
25
+ }
@@ -125,7 +125,7 @@ async function checkContentDisk() {
125
125
  }
126
126
 
127
127
  async function checkStalePages() {
128
- const pages = await listPages().catch(() => []);
128
+ const pages = await listPages({render: false}).catch(() => []);
129
129
  const cutoff = Date.now() - STALE_PAGE_DAYS * 24 * 60 * 60 * 1000;
130
130
  const stale = pages.filter(p => {
131
131
  const updated = p.updatedAt ? new Date(p.updatedAt).getTime() : 0;
@@ -141,7 +141,7 @@ async function checkStalePages() {
141
141
  }
142
142
 
143
143
  async function checkMissingMeta() {
144
- const pages = await listPages().catch(() => []);
144
+ const pages = await listPages({render: false}).catch(() => []);
145
145
  const missing = pages.filter(p => !p.description || String(p.description).trim() === '');
146
146
  return {
147
147
  id: 'missing-meta',
@@ -96,6 +96,34 @@ export function htmlHash(html) {
96
96
  return createHash('sha1').update(html).digest('base64');
97
97
  }
98
98
 
99
+ // A page served from the response cache is the SAME string on every hit, and
100
+ // hashing it (sha1 over the whole document) was half of what a hit cost. The
101
+ // hash is kept per cache key beside the string it was made from, and used only
102
+ // while the caller hands in that very string - a page rendered again is a new
103
+ // string, fails the comparison (identity first, so a hit compares nothing) and
104
+ // is hashed afresh. So a kept hash can never describe other HTML. Dropped
105
+ // whenever anything leaves the response cache, so it does not hold a page the
106
+ // cache has let go.
107
+ const HASH_MEMO_MAX = 2000;
108
+ /** @type {Map<string, {html: string, hash: string}>} */
109
+ const _hashMemo = new Map();
110
+ cache.onInvalidate(() => _hashMemo.clear());
111
+
112
+ function memoisedHash(key, html) {
113
+ const kept = _hashMemo.get(key);
114
+ if (kept && kept.html === html) return kept.hash;
115
+ const hash = htmlHash(html);
116
+ _hashMemo.delete(key);
117
+ _hashMemo.set(key, {html, hash});
118
+ while (_hashMemo.size > HASH_MEMO_MAX) _hashMemo.delete(_hashMemo.keys().next().value);
119
+ return hash;
120
+ }
121
+
122
+ /** For tests: how many hashes are kept. */
123
+ export function _hashMemoSize() {
124
+ return _hashMemo.size;
125
+ }
126
+
99
127
  /**
100
128
  * Send a rendered page: cache-control, an ETag, a 304 when the browser
101
129
  * already has it, and - for a response that came from the response cache -
@@ -114,11 +142,12 @@ export function htmlHash(html) {
114
142
  * @param {import('fastify').FastifyRequest} request
115
143
  * @param {import('fastify').FastifyReply} reply
116
144
  * @param {string} html
117
- * @param {{cached?: boolean}} [opts] - cached: the HTML came from the response cache, so its
118
- * compressed form may be kept too
145
+ * @param {{cached?: boolean, key?: string}} [opts] - cached: the HTML came from the response cache, so its
146
+ * compressed form may be kept too; key: the response-cache key it came from, so its hash is
147
+ * kept as well (only with `cached`)
119
148
  */
120
- export async function sendHtml(request, reply, html, {cached = false} = {}) {
121
- const hash = htmlHash(html);
149
+ export async function sendHtml(request, reply, html, {cached = false, key = ''} = {}) {
150
+ const hash = cached && key && cache.isEnabled() ? memoisedHash(key, html) : htmlHash(html);
122
151
  const encoding = cached && Buffer.byteLength(html) >= COMPRESS_THRESHOLD
123
152
  ? pickEncoding(request.headers['accept-encoding'])
124
153
  : null;
@@ -173,7 +173,7 @@ function toolTagFor(kind, raw, names) {
173
173
 
174
174
  /** The items of a kind as `{id, tags, raw}`, through the same listers the routes use. */
175
175
  async function items(kind) {
176
- if (kind === 'pages') return (await listPages()).map(p => ({id: p.urlPath, tags: cleanTagList(p.adminTags), raw: p}));
176
+ if (kind === 'pages') return (await listPages({render: false})).map(p => ({id: p.urlPath, tags: cleanTagList(p.adminTags), raw: p}));
177
177
  if (kind === 'collections') {
178
178
  const map = readRegistry().collectionTags;
179
179
  return (await listCollections()).map(c => ({id: c.slug, tags: cleanTagList(map[c.slug]), raw: c}));
@@ -2071,6 +2071,32 @@ export function unindentShortTags(markdown) {
2071
2071
  return lines.map((l, i) => (layout[i].cut ? l.slice(layout[i].cut) : l)).join('\n');
2072
2072
  }
2073
2073
 
2074
+ const SCRUB_PLACEHOLDER = /\x02SC(\d+):(\d+)\x03/g;
2075
+ const SHIELD_PLACEHOLDER = /DMSCT(\d+)x(\d+)x/g;
2076
+
2077
+ /**
2078
+ * The two patterns a shortcode name is matched with - self-closing and wrapping -
2079
+ * built once per name. `bounded`: the name must end there ([map] is not [map-guess]).
2080
+ */
2081
+ const _shortcodePatterns = new Map();
2082
+ function shortcodePatterns(name, bounded) {
2083
+ const key = `${bounded ? 'b' : 'u'}:${name}`;
2084
+ let kept = _shortcodePatterns.get(key);
2085
+ if (!kept) {
2086
+ const end = bounded ? '(?=[\\s/\\]])' : '';
2087
+ kept = {
2088
+ selfClosing: new RegExp(`\\[${name}${end}([^\\]]*)\\s*\\/\\]`, 'gi'),
2089
+ wrapping: new RegExp(`\\[${name}${end}([^\\]]*)\\]([\\s\\S]*?)\\[\\/${name}\\]`, 'gi'),
2090
+ // What must be in the text (lower-cased) for either to match at all.
2091
+ needle: `[${String(name).toLowerCase()}`,
2092
+ // A name with pattern characters in it cannot be looked for as plain text.
2093
+ plain: /^[\w-]+$/.test(String(name))
2094
+ };
2095
+ _shortcodePatterns.set(key, kept);
2096
+ }
2097
+ return kept;
2098
+ }
2099
+
2074
2100
  export function scrubCodeRegions(markdown, preserveIndented = false) {
2075
2101
  const callId = ++scrubCallId;
2076
2102
  const store = [];
@@ -2095,7 +2121,12 @@ export function scrubCodeRegions(markdown, preserveIndented = false) {
2095
2121
  // A stored region can hold an earlier placeholder of this same call - an indented
2096
2122
  // line (preserveIndented) holding an inline <code> - so what comes back is
2097
2123
  // restored too. Restoring it once left "SC1:0" on the page.
2098
- const restore = (s) => s.replace(new RegExp(`\\x02SC${callId}:(\\d+)\\x03`, 'g'), (_, i) => restore(store[parseInt(i, 10)]));
2124
+ // One shared pattern, the call checked in the callback: a pattern built per call
2125
+ // (its id in the source) is compiled afresh every time and never optimised, and
2126
+ // this runs many times for every page drawn. Another call's placeholders are left alone.
2127
+ const restore = (s) => (store.length
2128
+ ? s.replace(SCRUB_PLACEHOLDER, (m, id, i) => (Number(id) === callId ? restore(store[Number(i)]) : m))
2129
+ : s);
2099
2130
  return {scrubbed, restore};
2100
2131
  }
2101
2132
 
@@ -2119,7 +2150,7 @@ function markedBody(md) {
2119
2150
  const {scrubbed, restore} = scrubCodeRegions(text);
2120
2151
  const shielded = restore(scrubbed.replace(ATTR_TAG, (tag) => { held.push(tag); return `DMSCT${id}x${held.length - 1}x`; }));
2121
2152
  if (!held.length) return marked.parse(text);
2122
- return marked.parse(shielded).replace(new RegExp(`DMSCT${id}x(\\d+)x`, 'g'), (_, i) => held[Number(i)]);
2153
+ return marked.parse(shielded).replace(SHIELD_PLACEHOLDER, (m, call, i) => (Number(call) === id ? held[Number(i)] : m));
2123
2154
  }
2124
2155
 
2125
2156
  /**
@@ -2158,20 +2189,19 @@ async function processPluginShortcodes(markdown, user = null) {
2158
2189
  return res + input.slice(lastEnd);
2159
2190
  }
2160
2191
 
2192
+ // Most pages use none of most plugins' shortcodes: a plain-text look for "[name"
2193
+ // first, so a page pays two pattern scans only for the names it holds.
2194
+ let lower = result.toLowerCase();
2161
2195
  for (const {name, handler} of processors) {
2196
+ // The name must end there: [map] is not [map-guess].
2197
+ const patterns = shortcodePatterns(name, true);
2198
+ if (patterns.plain && !lower.includes(patterns.needle)) continue;
2199
+ const before = result;
2162
2200
  // Self-closing: [name attrs /]
2163
- result = await replaceAsync(
2164
- result,
2165
- // The name must end there: [map] is not [map-guess].
2166
- new RegExp(`\\[${name}(?=[\\s/\\]])([^\\]]*)\\s*\\/\\]`, 'gi'),
2167
- (_, attrStr) => handler(attrStr, null, context)
2168
- );
2201
+ result = await replaceAsync(result, patterns.selfClosing, (_, attrStr) => handler(attrStr, null, context));
2169
2202
  // Wrapping: [name attrs]...[/name]
2170
- result = await replaceAsync(
2171
- result,
2172
- new RegExp(`\\[${name}(?=[\\s/\\]])([^\\]]*)\\]([\\s\\S]*?)\\[\\/${name}\\]`, 'gi'),
2173
- (_, attrStr, body) => handler(attrStr, body, context)
2174
- );
2203
+ result = await replaceAsync(result, patterns.wrapping, (_, attrStr, body) => handler(attrStr, body, context));
2204
+ if (result !== before) lower = result.toLowerCase();
2175
2205
  }
2176
2206
  return restore(result);
2177
2207
  }
@@ -2189,15 +2219,15 @@ function processEffectsBlocks(markdown) {
2189
2219
  const {scrubbed, restore} = scrubCodeRegions(markdown);
2190
2220
  let result = scrubbed;
2191
2221
 
2222
+ // As with plugin shortcodes: look for "[name" as plain text before scanning with the patterns.
2223
+ let lower = result.toLowerCase();
2192
2224
  function apply(name, handler) {
2193
- result = result.replace(
2194
- new RegExp(`\\[${name}([^\\]]*)\\s*\\/\\]`, 'gi'),
2195
- (_, attrStr) => handler(attrStr, null)
2196
- );
2197
- result = result.replace(
2198
- new RegExp(`\\[${name}([^\\]]*)\\]([\\s\\S]*?)\\[\\/${name}\\]`, 'gi'),
2199
- (_, attrStr, body) => handler(attrStr, body)
2200
- );
2225
+ const patterns = shortcodePatterns(name, false);
2226
+ if (patterns.plain && !lower.includes(patterns.needle)) return;
2227
+ const before = result;
2228
+ result = result.replace(patterns.selfClosing, (_, attrStr) => handler(attrStr, null));
2229
+ result = result.replace(patterns.wrapping, (_, attrStr, body) => handler(attrStr, body));
2230
+ if (result !== before) lower = result.toLowerCase();
2201
2231
  }
2202
2232
 
2203
2233
  apply('counter', (attrStr) => {
@@ -2230,6 +2260,7 @@ function processEffectsBlocks(markdown) {
2230
2260
  // A single inline character also accepts the concise, unpaired form.
2231
2261
  result = result.replace(/\[celebrate([^\]]*)\]/gi, (match, attrs) =>
2232
2262
  parseShortcodeAttrs(attrs).character ? renderCelebration(attrs) : match);
2263
+ lower = result.toLowerCase();
2233
2264
 
2234
2265
  // Named celebration shortcodes - sugar for [celebrate theme="..."] with override support
2235
2266
  ['christmas', 'halloween', 'st-patricks', 'valentines', 'guy-fawkes',
@@ -5406,7 +5437,10 @@ export function sanitizePageHtml(html) {
5406
5437
  */
5407
5438
  export async function parseMarkdown(raw, opts = {}) {
5408
5439
  opts = {...opts, [SLOT_BUDGET]:opts[SLOT_BUDGET] || {remaining:500}};
5409
- const {data, content: written} = matter(raw);
5440
+ // Options passed on purpose: without them gray-matter keeps every string it is
5441
+ // given for the life of the process - each editor preview, each version of each
5442
+ // page - and hands every caller the same `data` object.
5443
+ const {data, content: written} = matter(raw, {});
5410
5444
  const content = unindentShortTags(written);
5411
5445
 
5412
5446
  // Pipeline (earlier stages are already HTML when later ones run; keep in step
@@ -78,7 +78,7 @@ export function titleFromSlug(slug) {
78
78
  */
79
79
  export function pageMeta(slug, source) {
80
80
  let data = {};
81
- try { data = matter(source).data || {}; } catch { data = {}; }
81
+ try { data = matter(source, {}).data || {}; } catch { data = {}; }
82
82
  const title = typeof data.title === 'string' && data.title.trim() ? data.title.trim() : titleFromSlug(slug);
83
83
  const order = Number.isFinite(Number(data.order)) && data.order !== null && data.order !== ''
84
84
  ? Number(data.order)
@@ -123,7 +123,7 @@ const SANITIZE = {
123
123
  */
124
124
  export function renderGuide(source) {
125
125
  let body = source;
126
- try { body = matter(source).content; } catch { /* no frontmatter */ }
126
+ try { body = matter(source, {}).content; } catch { /* no frontmatter */ }
127
127
  return sanitizeHtml(marked.parse(body, {gfm: true, async: false}), SANITIZE);
128
128
  }
129
129