domma-cms 1.60.0 → 1.61.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/CLAUDE.md +1 -1
  3. package/admin/index.html +7 -1
  4. package/admin/js/app.js +2 -2
  5. package/admin/js/lib/ai-assist.js +1 -1
  6. package/admin/js/templates/ai-assist-builders.html +29 -0
  7. package/admin/js/templates/docs/usage-chat.html +5 -2
  8. package/admin/js/views/ai-assist-builders.js +1 -0
  9. package/admin/js/views/ai-assist-settings.js +1 -1
  10. package/admin/js/views/index.js +1 -1
  11. package/admin/js/views/login.js +1 -1
  12. package/docs/DommaFeatures.md +4 -2
  13. package/docs/cache.md +24 -0
  14. package/docs/configuration.md +1 -1
  15. package/docs/plugin-development.md +26 -0
  16. package/package.json +1 -1
  17. package/public/js/effects.js +1 -1
  18. package/public/js/site.js +1 -1
  19. package/server/config.js +37 -0
  20. package/server/routes/api/admin-assistant.js +11 -3
  21. package/server/routes/api/auth.js +4 -1
  22. package/server/routes/api/chats.js +3 -1
  23. package/server/routes/api/components.js +2 -1
  24. package/server/routes/api/content-api.js +1 -1
  25. package/server/routes/api/content-public.js +2 -2
  26. package/server/routes/api/content-writes.js +1 -1
  27. package/server/routes/api/effects.js +4 -9
  28. package/server/routes/api/layouts.js +1 -1
  29. package/server/routes/api/pages.js +6 -5
  30. package/server/routes/api/search.js +3 -3
  31. package/server/routes/api/sidebar-counts.js +2 -0
  32. package/server/routes/api/theme.js +4 -7
  33. package/server/routes/api/views.js +2 -1
  34. package/server/routes/public.js +4 -4
  35. package/server/server.js +14 -7
  36. package/server/services/aiAssist.js +67 -0
  37. package/server/services/cache/adminChange.js +28 -0
  38. package/server/services/chat/ai.js +1 -1
  39. package/server/services/content.js +73 -10
  40. package/server/services/effectsSettings.js +25 -0
  41. package/server/services/health.js +2 -2
  42. package/server/services/hooks.js +1 -1
  43. package/server/services/htmlResponse.js +33 -4
  44. package/server/services/listTags.js +1 -1
  45. package/server/services/markdown.js +56 -22
  46. package/server/services/pluginGuides.js +2 -2
  47. package/server/services/plugins.js +34 -3
  48. package/server/services/projects.js +39 -4
  49. package/server/services/publicAssets.js +488 -0
  50. package/server/services/renderer.js +10 -8
  51. package/server/services/seo/index.js +3 -3
  52. package/server/services/sitemap.js +1 -1
  53. package/server/services/themeSettings.js +11 -0
  54. package/server/services/versions.js +2 -2
  55. package/server/services/wpImport/index.js +2 -2
  56. package/server/templates/page.html +3 -2
@@ -20,11 +20,10 @@ import {
20
20
  requirePermission as defaultRequirePermission
21
21
  } from '../../middleware/auth.js';
22
22
  import {
23
- compileCustomThemeCss,
23
+ adminThemePresentation,
24
24
  compileOverrideCss,
25
25
  getThemeTokenMap,
26
26
  listCustomThemes,
27
- resolveThemeClasses,
28
27
  listThemeIds,
29
28
  listThemeTokens,
30
29
  loadThemeConfig,
@@ -121,10 +120,7 @@ export async function themeRoutes(fastify, opts = {}) {
121
120
  if (reply.sent) return reply;
122
121
  }
123
122
  const id = asked || saved;
124
- const {dommaTheme, customClass} = resolveThemeClasses(id, cfg);
125
- return {theme: id, dommaTheme, customClass, css: customClass ? compileCustomThemeCss([id]) : '',
126
- font: {family: cfg.font?.adminFamily || '', stylesheet: cfg.font?.stylesheet || ''},
127
- sidebar: cfg.adminSidebar !== false};
123
+ return adminThemePresentation(id, cfg);
128
124
  });
129
125
 
130
126
  /*
@@ -146,7 +142,8 @@ export async function themeRoutes(fastify, opts = {}) {
146
142
  }
147
143
  });
148
144
 
149
- fastify.post('/theme/compile', canRead, async (request, reply) => {
145
+ // Compiles what it is sent and returns it: not a change (cache/adminChange.js).
146
+ fastify.post('/theme/compile', {...canRead, config: {keepsCache: true}}, async (request, reply) => {
150
147
  const candidate = request.body || {};
151
148
  const errors = validateThemeConfig(candidate, {remembered: rememberedIds(await loadThemeConfig())});
152
149
  if (errors.length) return reply.code(400).send({error: errors.join('; '), validation: errors});
@@ -181,7 +181,8 @@ export async function viewsRoutes(fastify) {
181
181
  // user may see (project scope), and it runs as an admin preview - no
182
182
  // row-level filter, the same as executing a saved view from the admin.
183
183
  // `_` is outside the slug alphabet, so no view can shadow the route.
184
- fastify.post('/views/_preview', canUpdate, async (request, reply) => {
184
+ // Runs an unsaved view and returns its rows: not a change (cache/adminChange.js).
185
+ fastify.post('/views/_preview', {...canUpdate, config: {keepsCache: true}}, async (request, reply) => {
185
186
  const {source, connection = 'default', stages = [], limit} = request.body || {};
186
187
  if (typeof source !== 'string' || !source) return reply.status(400).send({error: 'Choose a source collection'});
187
188
  if (!Array.isArray(stages)) return reply.status(400).send({error: 'stages must be an array'});
@@ -17,7 +17,7 @@ import {validSessionCookie, newSessionCookie} from '../services/shortcodeExpiry.
17
17
  import {touchPreviewLink, verifyPreviewToken} from '../services/previewLinks.js';
18
18
  import {viewerRoleKey} from '../services/userRoles.js';
19
19
  import {hooks} from '../services/hooks.js';
20
- import {getConfig} from '../config.js';
20
+ import {readConfig} from '../config.js';
21
21
  import * as cache from '../services/cache/index.js';
22
22
 
23
23
  /**
@@ -34,7 +34,7 @@ import * as cache from '../services/cache/index.js';
34
34
  * @returns {string} e.g. 'https://example.com' (no trailing slash)
35
35
  */
36
36
  export function getBaseUrl(request) {
37
- const override = (getConfig('site') || {}).baseUrl;
37
+ const override = (readConfig('site') || {}).baseUrl;
38
38
  if (override) return override.toString().trim().replace(/\/+$/, '');
39
39
  const host = request.host || request.headers.host || request.hostname;
40
40
  return `${request.protocol}://${host}`;
@@ -283,7 +283,7 @@ export async function publicRoutes(fastify) {
283
283
  const pageMtime = await getPageMtime(page.urlPath || urlPath);
284
284
  const cacheKey = `page:${urlPath}:m${pageMtime}:role${viewerRoleKey(userObj)}:o${baseUrl}`;
285
285
  // The footer's blocks (site.json footer.blocks) are on every page: editing one must reach the cached pages.
286
- const cacheTags = [`page:${urlPath}`, ...(page.cacheTags || []), 'nav', 'site', ...footerBlockCacheTags(getConfig('site'))];
286
+ const cacheTags = [`page:${urlPath}`, ...(page.cacheTags || []), 'nav', 'site', ...footerBlockCacheTags(readConfig('site'))];
287
287
  // x-dm-cache says which way this response went - hit, miss, off (the
288
288
  // cache is disabled) or bypass (a draft, an expiring or a per-viewer
289
289
  // page, never cached) - so a slow page can be read from the browser.
@@ -294,7 +294,7 @@ export async function publicRoutes(fastify) {
294
294
  reply.header('x-dm-cache', status);
295
295
  hooks.emit('content:pageViewed', {urlPath, title: page.title || ''});
296
296
  // cached: this HTML is shared, so its compressed body is kept too (htmlResponse.js).
297
- return sendHtml(request, reply, html, {cached: true});
297
+ return sendHtml(request, reply, html, {cached: true, key: cacheKey});
298
298
  });
299
299
  }
300
300
 
package/server/server.js CHANGED
@@ -19,7 +19,7 @@ import path from 'path';
19
19
  import fs from 'fs/promises';
20
20
  import {fileURLToPath} from 'url';
21
21
  import {createRequire} from 'module';
22
- import {config, getConfig} from './config.js';
22
+ import {config, getConfig, readConfig} from './config.js';
23
23
  import {getLoadedPlugins, getPluginSettings, recordPluginLoadFailure, registerPlugins} from './services/plugins.js';
24
24
  import {notifierFor} from './services/notify.js';
25
25
  import {grantAdminToolDefaults, grantPermissionOnce, load as loadRoles, seed as seedRoles} from './services/roles.js';
@@ -34,6 +34,7 @@ import {parseTrustProxy} from './services/network.js';
34
34
  import {requestSerializer} from './services/request-log.js';
35
35
  import {rateLimitOptions} from './services/rate-limit.js';
36
36
  import {registerAdminAssets} from './services/adminAssets.js';
37
+ import {registerPublicAssets} from './services/publicAssets.js';
37
38
  import {isActiveType, servedType} from './services/mediaTypes.js';
38
39
  import {connectSources, frameSources, imageSources} from './services/embedCsp.js';
39
40
  import {cspSources as gaSources} from './services/googleAnalytics.js';
@@ -131,7 +132,7 @@ await app.register(helmet, {
131
132
  scriptSrc: ["'self'", "'unsafe-inline'", "'wasm-unsafe-eval'", 'https://cdn.jsdelivr.net', gaSources('script')],
132
133
  styleSrc: ["'self'", "'unsafe-inline'", 'https://cdn.jsdelivr.net', 'https://fonts.googleapis.com'],
133
134
  // config/server.json csp.imgHosts: other sites a page may show images from (services/embedCsp.js).
134
- imgSrc: [() => imageSources(getConfig('server').csp?.imgHosts).join(' '), gaSources('img')],
135
+ imgSrc: [() => imageSources(readConfig('server').csp?.imgHosts).join(' '), gaSources('img')],
135
136
  fontSrc: ["'self'", 'data:', 'https://fonts.gstatic.com'],
136
137
  // config/server.json csp.connectHosts: other sites a page may read from (services/embedCsp.js).
137
138
  connectSrc: [...connectSources(serverConfig.csp?.connectHosts), gaSources('connect')],
@@ -399,11 +400,11 @@ try {
399
400
  * tag. Runs on onResponse, after the reply has gone, so it costs the caller
400
401
  * nothing - and a memory-driver clear is two Map resets.
401
402
  */
402
- const CACHE_CLEARING_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
403
+ // A route that changes nothing a page could show opts out with `config: {keepsCache: true}`
404
+ // (services/cache/adminChange.js) - the admin's 30-second chat check-in was emptying the cache.
405
+ const {isCacheClearingChange} = await import('./services/cache/adminChange.js');
403
406
  app.addHook('onResponse', async (request, reply) => {
404
- if (!CACHE_CLEARING_METHODS.has(request.method)) return;
405
- if (reply.statusCode >= 400) return;
406
- if (!request.user) return;
407
+ if (!isCacheClearingChange(request, reply)) return;
407
408
  try {
408
409
  await cache.clear();
409
410
  } catch (err) {
@@ -442,6 +443,10 @@ await app.register(staticPlugin, {
442
443
  }
443
444
  });
444
445
 
446
+ // Fingerprinted public assets are cached for good, and the site's custom CSS is
447
+ // served as a file - services/publicAssets.js.
448
+ const publicAssets = registerPublicAssets(app, {root: ROOT, dommaDist: DOMMA_DIST, pluginsDir});
449
+
445
450
  // Serve plugin admin/ and public/ subdirs only - block plugin.js, config.js, data/, etc.
446
451
  await app.register(async function pluginStaticScope(instance) {
447
452
  instance.addHook('onRequest', (request, reply, done) => {
@@ -784,13 +789,15 @@ await registerPlugins(app);
784
789
  const {isAiAssist} = await import('./services/aiAssist.js');
785
790
  if (isAiAssist()) {
786
791
  const url = isToolEnabled('launchers') ? '#/ai-assist/chats' : '#/ai-assist/settings';
787
- registerSidebarItem({item: {source: 'Built in', text: 'AI Assist', url, icon: 'sparkles', permission: isToolEnabled('launchers') ? 'chats' : 'settings',
792
+ // Under Data by default; an admin's own placement in the menu still wins.
793
+ registerSidebarItem({folder: 'data', item: {source: 'Built in', text: 'AI Assist', url, icon: 'sparkles', permission: isToolEnabled('launchers') ? 'chats' : 'settings',
788
794
  ...(isToolEnabled('launchers') && {countUrl: '/api/chats/_count'})}});
789
795
  hideSidebarUrl('#/launchers/chats', url);
790
796
  }
791
797
  }
792
798
  // After the plugins' onRequest hooks - see serve() in services/adminAssets.js.
793
799
  adminAssets.serve();
800
+ publicAssets.serve();
794
801
 
795
802
  // ---------------------------------------------------------------------------
796
803
  // Public Plugin Routes (root-level, before catch-all)
@@ -50,7 +50,74 @@ export function chatsHome() {
50
50
  return isAiAssist() ? AI_ASSIST_CHATS : LAUNCHER_CHATS;
51
51
  }
52
52
 
53
+ // ---------------------------------------------------------------------------
54
+ // Builders: AI chats of other Tools, listed on AI Assist's Builders tab
55
+ // ---------------------------------------------------------------------------
56
+
57
+ /**
58
+ * plugin name -> its builder entry. A plugin declares one in plugin.json:
59
+ *
60
+ * "admin": {"aiAssist": {"label": "Site Builder", "url": "#/plugins/ai-site-builder",
61
+ * "list": "/api/plugins/ai-site-builder/conversations", "permission": "ai-builder",
62
+ * "character": "dot"}}
63
+ *
64
+ * `url` must be the plugin's own admin screen and `list` its own API: AI Assist only
65
+ * links to the plugin and reads its list with the viewer's own token, so the plugin's
66
+ * routes still decide what each person sees. Registered by the loader for enabled plugins.
67
+ */
68
+ const builders = new Map();
69
+ const SLUG = /^[a-z0-9][a-z0-9-]{0,63}$/;
70
+
71
+ /**
72
+ * What is wrong with a plugin's `admin.aiAssist` block, or ''.
73
+ *
74
+ * @param {string} plugin - the manifest name
75
+ * @param {object} block
76
+ */
77
+ export function aiAssistBuilderProblem(plugin, block) {
78
+ if (!SLUG.test(String(plugin || ''))) return 'invalid plugin name';
79
+ if (!block || typeof block !== 'object' || Array.isArray(block)) return 'admin.aiAssist must be an object';
80
+ const own = (re) => new RegExp(re.replace('<p>', plugin.replace(/-/g, '\\-')));
81
+ if (typeof block.label !== 'string' || !block.label.trim() || block.label.length > 40) return 'label must be text of up to 40 characters';
82
+ if (typeof block.url !== 'string' || !own('^#/plugins/<p>(/[a-z0-9-]+)*$').test(block.url)) return `url must be this plugin's own screen (#/plugins/${plugin}...)`;
83
+ if (typeof block.list !== 'string' || !own('^/api/plugins/<p>(/[a-z0-9-]+)+$').test(block.list)) return `list must be this plugin's own API (/api/plugins/${plugin}/...)`;
84
+ if (typeof block.permission !== 'string' || !/^[a-z0-9][a-z0-9.-]{0,79}$/.test(block.permission)) return 'permission is required';
85
+ if (block.character !== undefined && (typeof block.character !== 'string' || !SLUG.test(block.character))) return 'character must be a character id';
86
+ return '';
87
+ }
88
+
89
+ /** Register a loaded plugin's builder entry; returns the problem ('' when registered). */
90
+ export function registerAiAssistBuilder(plugin, block) {
91
+ const problem = aiAssistBuilderProblem(plugin, block);
92
+ if (problem) return problem;
93
+ builders.set(plugin, {plugin, label: block.label.trim(), url: block.url, list: block.list, permission: block.permission,
94
+ ...(block.character ? {character: block.character} : {})});
95
+ return '';
96
+ }
97
+
98
+ /** The plugin was switched off: its chats leave the Builders tab at once, not at the next restart. */
99
+ export function unregisterAiAssistBuilder(plugin) {
100
+ return builders.delete(String(plugin || ''));
101
+ }
102
+
103
+ /**
104
+ * The builders this viewer may open: they hold the entry's permission (the resource or
105
+ * any action of it), and a confined user's admin scope covers its screen.
106
+ *
107
+ * @param {{permissions: string[], adminScope?: {allow?: string[]}|null}} access - services/adminAccess.js adminAccessFor()
108
+ * @param {(prefix: string, path: string) => boolean} covers - services/pluginScope.js covers()
109
+ */
110
+ export function aiAssistBuildersFor(access, covers) {
111
+ const perms = access?.permissions || [];
112
+ const holds = (p) => perms.includes(p) || perms.some(x => x.startsWith(`${p}.`)) || perms.includes(p.split('.')[0]);
113
+ const scope = access?.adminScope;
114
+ const inScope = (url) => !scope || (scope.allow || []).some(prefix => covers(prefix, url.slice(1)));
115
+ return [...builders.values()].filter(b => holds(b.permission) && inScope(b.url))
116
+ .map(({plugin, label, url, list, character}) => ({plugin, label, url, list, ...(character ? {character} : {})}));
117
+ }
118
+
53
119
  /** Tests only: forget every owner. */
54
120
  export function _resetAiAssistForTests() {
55
121
  owners.clear();
122
+ builders.clear();
56
123
  }
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Does this finished request count as an admin change that clears the
3
+ * response cache? (The safety net in server.js - see the comment there.)
4
+ *
5
+ * A signed-in, successful POST/PUT/PATCH/DELETE does - unless its route says
6
+ * it changes nothing a page could show: `config: {keepsCache: true}`. The
7
+ * admin's "available for chats" check-in is one, sent every 30 seconds from
8
+ * every admin screen; counted as a change, one signed-in person emptied the
9
+ * whole cache twice a minute and no page was ever served from it.
10
+ *
11
+ * Only for a route whose effect lives outside every rendered page. When in
12
+ * doubt leave it off: a cache cleared too often is slow, a cache not cleared
13
+ * is wrong.
14
+ */
15
+ const CLEARING_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
16
+
17
+ /**
18
+ * @param {{method: string, user?: object, routeOptions?: {config?: {keepsCache?: boolean}}}} request
19
+ * @param {{statusCode: number}} reply
20
+ * @returns {boolean}
21
+ */
22
+ export function isCacheClearingChange(request, reply) {
23
+ if (!CLEARING_METHODS.has(request.method)) return false;
24
+ if (reply.statusCode >= 400) return false;
25
+ if (!request.user) return false;
26
+ if (request.routeOptions?.config?.keepsCache === true) return false;
27
+ return true;
28
+ }
@@ -22,7 +22,7 @@ const REGISTRY = Symbol.for('domma.launcher-chat');
22
22
  /** What a visitor will wait for an answer. */
23
23
  const ASK_MS = 25_000;
24
24
  /** Knowledge and chat sent at most: a reply is a few sentences, the prompt need not be a book. */
25
- export const KNOWLEDGE_CHARS = 16_000;
25
+ export const KNOWLEDGE_CHARS = 48_000;
26
26
  export const HISTORY_MESSAGES = 10;
27
27
  /** After the Manager says no (no licence, the month's answers used), ask again only after this. */
28
28
  const BACKOFF_MS = {403: 10 * 60_000, 429: 30 * 60_000, other: 60_000};
@@ -35,6 +35,9 @@ const MEDIA_DIR = config.content.mediaDir;
35
35
  // the hooks do not know about. Entries are cloned in and out, so a caller
36
36
  // that edits the object it was handed edits its own copy.
37
37
  const PAGE_MEMO_MAX = 500;
38
+ /** listPages() raises the limit to the number of pages, up to this. */
39
+ const PAGE_MEMO_CEILING = 5000;
40
+ let _pageMemoMax = PAGE_MEMO_MAX;
38
41
  const PAGE_MEMO_TTL_MS = 5 * 60 * 1000;
39
42
  /** @type {Map<string, {mtimeMs: number, size: number, at: number, page: object}>} */
40
43
  const _pageMemo = new Map();
@@ -43,11 +46,13 @@ cache.onInvalidate(() => _pageMemo.clear());
43
46
 
44
47
  /** For the cache status route and tests: how the memo is doing. */
45
48
  export function pageMemoStats() {
46
- return {size: _pageMemo.size, max: PAGE_MEMO_MAX, hits: _pageMemoStats.hits, misses: _pageMemoStats.misses};
49
+ return {size: _pageMemo.size, max: _pageMemoMax, hits: _pageMemoStats.hits, misses: _pageMemoStats.misses};
47
50
  }
48
51
 
49
52
  export function _resetPageMemoForTests() {
50
53
  _pageMemo.clear();
54
+ _summaryMemo.clear();
55
+ _pageMemoMax = PAGE_MEMO_MAX;
51
56
  _pageMemoStats.hits = 0;
52
57
  _pageMemoStats.misses = 0;
53
58
  }
@@ -69,7 +74,7 @@ async function readPageMemo(filePath) {
69
74
  } catch {
70
75
  return page; // something in it cannot be cloned: hand it over unmemoised
71
76
  }
72
- while (_pageMemo.size > PAGE_MEMO_MAX) _pageMemo.delete(_pageMemo.keys().next().value);
77
+ while (_pageMemo.size > _pageMemoMax) _pageMemo.delete(_pageMemo.keys().next().value);
73
78
  return page;
74
79
  }
75
80
 
@@ -77,21 +82,80 @@ async function readPageMemo(filePath) {
77
82
  // Pages
78
83
  // ---------------------------------------------------------------------------
79
84
 
85
+ // ---------------------------------------------------------------------------
86
+ // Page summaries (front matter + source, nothing rendered)
87
+ // ---------------------------------------------------------------------------
88
+ //
89
+ // A page's front matter and Markdown source depend on its file and nothing
90
+ // else, so they are kept per file and checked against the file's mtime and
91
+ // size on every read - no TTL, and correct with the response cache on or off.
92
+ // Emptied with the response cache all the same, so two saves inside one mtime
93
+ // tick cannot leave the first one's values behind. Cloned out, so a caller
94
+ // that edits a row edits its own copy.
95
+ /** @type {Map<string, {mtimeMs: number, size: number, page: object}>} */
96
+ const _summaryMemo = new Map();
97
+ cache.onInvalidate(() => _summaryMemo.clear());
98
+
99
+ async function readPageSummary(filePath) {
100
+ const st = await fs.stat(filePath);
101
+ const kept = _summaryMemo.get(filePath);
102
+ if (kept && kept.mtimeMs === st.mtimeMs && kept.size === st.size) return structuredClone(kept.page);
103
+ const raw = await fs.readFile(filePath, 'utf8');
104
+ // Options passed on purpose: without them gray-matter keeps every string it
105
+ // has ever parsed (and hands the same `data` object to every caller).
106
+ const {data, content} = matter(raw, {});
107
+ const page = {...data, urlPath: filePathToUrlPath(filePath), content};
108
+ try {
109
+ _summaryMemo.set(filePath, {mtimeMs: st.mtimeMs, size: st.size, page: structuredClone(page)});
110
+ } catch { /* something in the front matter cannot be cloned: read it each time */ }
111
+ return page;
112
+ }
113
+
80
114
  /**
81
115
  * Recursively list all .md files under PAGES_DIR.
82
- * Returns parsed page objects sorted by sortOrder then title.
116
+ * Returns page objects sorted by sortOrder then title.
117
+ *
118
+ * By default every page is parsed in full, as getPage() does - shortcodes,
119
+ * collections, sanitising - which is tens of milliseconds a page and seconds
120
+ * for a few hundred. A caller that reads only the front matter (title, status,
121
+ * layout, tags, seo, ...) or the Markdown source (`content`) asks for
122
+ * `{render: false}` and gets exactly those plus `urlPath` and
123
+ * `resolvedProject`: no `html`, `usedComponents`, `cacheTags`, `perViewer` or
124
+ * `hasExpiry`. Lists, badges, search and the sitemap all do.
125
+ *
126
+ * The rendered rows come through the parsed-page memo while the response
127
+ * cache is on, so a second listing does not parse every page again.
83
128
  *
129
+ * @param {{render?: boolean}} [opts]
84
130
  * @returns {Promise<object[]>}
85
131
  */
86
- export async function listPages() {
132
+ export async function listPages({render = true} = {}) {
87
133
  const files = await collectMdFiles(PAGES_DIR);
88
- const pages = await Promise.all(files.map(filePath => readPageFile(filePath)));
134
+ let pages;
135
+ if (!render) {
136
+ pages = await Promise.all(files.map(filePath => readPageSummary(filePath)));
137
+ // Forget pages that are gone, so the memo holds the site's pages and no more.
138
+ if (_summaryMemo.size > files.length) {
139
+ const live = new Set(files);
140
+ for (const key of _summaryMemo.keys()) if (!live.has(key)) _summaryMemo.delete(key);
141
+ }
142
+ } else if (cache.isEnabled()) {
143
+ // Every page must fit at once, or a listing evicts what it is about to read.
144
+ _pageMemoMax = Math.max(_pageMemoMax, Math.min(PAGE_MEMO_CEILING, files.length + 50));
145
+ pages = await Promise.all(files.map(filePath => readPageMemo(filePath)));
146
+ } else {
147
+ pages = await Promise.all(files.map(filePath => readPageFile(filePath)));
148
+ }
89
149
  pages.sort((a, b) => (a.sortOrder ?? 99) - (b.sortOrder ?? 99) || a.title.localeCompare(b.title));
90
- const {getProjectForPage} = await import('./projects.js');
150
+ // One read of the projects for the whole list, not one per page.
151
+ const {listProjects, projectForPage} = await import('./projects.js');
152
+ let projects = null;
91
153
  for (const page of pages) {
92
154
  const url = page.url || page.urlPath || '/';
93
155
  const explicit = (page.project === undefined) ? undefined : page.project;
94
- page.resolvedProject = await getProjectForPage(url, explicit);
156
+ if (typeof explicit === 'string' && explicit) { page.resolvedProject = explicit; continue; }
157
+ projects ??= await listProjects();
158
+ page.resolvedProject = projectForPage(projects, url, explicit);
95
159
  }
96
160
  return pages;
97
161
  }
@@ -327,9 +391,8 @@ export async function setPageFrontmatter(urlPath, patch) {
327
391
  throw err;
328
392
  }
329
393
  const raw = await fs.readFile(filePath, 'utf8');
330
- // gray-matter caches parses by input string and hands back the same
331
- // object: copy before changing it.
332
- const parsed = matter(raw);
394
+ // Options passed so gray-matter does not keep the string (see readPageSummary).
395
+ const parsed = matter(raw, {});
333
396
  const data = {...parsed.data};
334
397
  for (const [k, v] of Object.entries(patch || {})) {
335
398
  if (v === undefined) delete data[k];
@@ -0,0 +1,25 @@
1
+ /**
2
+ * The scroll-reveal defaults every public page's effects read
3
+ * (config/effects.json over the built-in defaults).
4
+ *
5
+ * Its own module so both the route that serves them and the renderer - which
6
+ * writes them into the page, saving every page view a request before its
7
+ * effects can start - read the same thing.
8
+ */
9
+ import {getConfig} from '../config.js';
10
+
11
+ export const EFFECTS_DEFAULTS = {respectMotion: true, defaultDuration: 600, defaultAnimation: 'fade', defaultThreshold: 0.1};
12
+
13
+ /** @returns {object} the defaults with config/effects.json over them */
14
+ export function loadEffectsSettings() {
15
+ try {
16
+ return {...EFFECTS_DEFAULTS, ...(getConfig('effects') || {})};
17
+ } catch {
18
+ return {...EFFECTS_DEFAULTS};
19
+ }
20
+ }
21
+
22
+ /** `window.__CMS_EFFECTS__ = {...};` for a page's inline script (public/js/effects.js reads it). */
23
+ export function effectsSettingsScript() {
24
+ return `window.__CMS_EFFECTS__ = ${JSON.stringify(loadEffectsSettings()).replace(/</g, '\\u003c')};`;
25
+ }
@@ -125,7 +125,7 @@ async function checkContentDisk() {
125
125
  }
126
126
 
127
127
  async function checkStalePages() {
128
- const pages = await listPages().catch(() => []);
128
+ const pages = await listPages({render: false}).catch(() => []);
129
129
  const cutoff = Date.now() - STALE_PAGE_DAYS * 24 * 60 * 60 * 1000;
130
130
  const stale = pages.filter(p => {
131
131
  const updated = p.updatedAt ? new Date(p.updatedAt).getTime() : 0;
@@ -141,7 +141,7 @@ async function checkStalePages() {
141
141
  }
142
142
 
143
143
  async function checkMissingMeta() {
144
- const pages = await listPages().catch(() => []);
144
+ const pages = await listPages({render: false}).catch(() => []);
145
145
  const missing = pages.filter(p => !p.description || String(p.description).trim() === '');
146
146
  return {
147
147
  id: 'missing-meta',
@@ -99,7 +99,7 @@ const _registeredSidebarItems = [];
99
99
  */
100
100
  const TOOL_GROUPS = new Map(Object.entries({
101
101
  Productivity: ['#/notes', '#/todo', '#/calculator', '#/plugins/calendar', '#/plugins/waypoint-pro', '#/plugins/sheets'],
102
- Communication: ['#/ai-assist/chats', '#/ai-assist/settings', '#/contacts', '#/plugins/contacts-pro', '#/plugins/mail-reader', '#/plugins/email-pro', '#/plugins/messages'],
102
+ Communication: ['#/contacts', '#/plugins/contacts-pro', '#/plugins/mail-reader', '#/plugins/email-pro', '#/plugins/messages'],
103
103
  Marketing: ['#/analytics', '#/seo', '#/plugins/analytics-pro/dashboard', '#/plugins/social', '#/plugins/feedback', '#/plugins/surveys', '#/plugins/campaigns-pro', '#/plugins/ecards'],
104
104
  Publishing: ['#/plugins/blog', '#/plugins/blog-pro', '#/plugins/docs', '#/plugins/theme-roller', '#/plugins/wiki', '#/plugins/wiki-pro', '#/plugins/scraper', '#/plugins/scraper-pro'],
105
105
  Business: ['#/plugins/shopping-cart', '#/plugins/booking', '#/plugins/booking-pro', '#/plugins/invoice', '#/job-board', '#/plugins/garage', '#/plugins/takeaway', '#/plugins/support', '#/plugins/onboard-pro'],
@@ -96,6 +96,34 @@ export function htmlHash(html) {
96
96
  return createHash('sha1').update(html).digest('base64');
97
97
  }
98
98
 
99
+ // A page served from the response cache is the SAME string on every hit, and
100
+ // hashing it (sha1 over the whole document) was half of what a hit cost. The
101
+ // hash is kept per cache key beside the string it was made from, and used only
102
+ // while the caller hands in that very string - a page rendered again is a new
103
+ // string, fails the comparison (identity first, so a hit compares nothing) and
104
+ // is hashed afresh. So a kept hash can never describe other HTML. Dropped
105
+ // whenever anything leaves the response cache, so it does not hold a page the
106
+ // cache has let go.
107
+ const HASH_MEMO_MAX = 2000;
108
+ /** @type {Map<string, {html: string, hash: string}>} */
109
+ const _hashMemo = new Map();
110
+ cache.onInvalidate(() => _hashMemo.clear());
111
+
112
+ function memoisedHash(key, html) {
113
+ const kept = _hashMemo.get(key);
114
+ if (kept && kept.html === html) return kept.hash;
115
+ const hash = htmlHash(html);
116
+ _hashMemo.delete(key);
117
+ _hashMemo.set(key, {html, hash});
118
+ while (_hashMemo.size > HASH_MEMO_MAX) _hashMemo.delete(_hashMemo.keys().next().value);
119
+ return hash;
120
+ }
121
+
122
+ /** For tests: how many hashes are kept. */
123
+ export function _hashMemoSize() {
124
+ return _hashMemo.size;
125
+ }
126
+
99
127
  /**
100
128
  * Send a rendered page: cache-control, an ETag, a 304 when the browser
101
129
  * already has it, and - for a response that came from the response cache -
@@ -114,11 +142,12 @@ export function htmlHash(html) {
114
142
  * @param {import('fastify').FastifyRequest} request
115
143
  * @param {import('fastify').FastifyReply} reply
116
144
  * @param {string} html
117
- * @param {{cached?: boolean}} [opts] - cached: the HTML came from the response cache, so its
118
- * compressed form may be kept too
145
+ * @param {{cached?: boolean, key?: string}} [opts] - cached: the HTML came from the response cache, so its
146
+ * compressed form may be kept too; key: the response-cache key it came from, so its hash is
147
+ * kept as well (only with `cached`)
119
148
  */
120
- export async function sendHtml(request, reply, html, {cached = false} = {}) {
121
- const hash = htmlHash(html);
149
+ export async function sendHtml(request, reply, html, {cached = false, key = ''} = {}) {
150
+ const hash = cached && key && cache.isEnabled() ? memoisedHash(key, html) : htmlHash(html);
122
151
  const encoding = cached && Buffer.byteLength(html) >= COMPRESS_THRESHOLD
123
152
  ? pickEncoding(request.headers['accept-encoding'])
124
153
  : null;
@@ -173,7 +173,7 @@ function toolTagFor(kind, raw, names) {
173
173
 
174
174
  /** The items of a kind as `{id, tags, raw}`, through the same listers the routes use. */
175
175
  async function items(kind) {
176
- if (kind === 'pages') return (await listPages()).map(p => ({id: p.urlPath, tags: cleanTagList(p.adminTags), raw: p}));
176
+ if (kind === 'pages') return (await listPages({render: false})).map(p => ({id: p.urlPath, tags: cleanTagList(p.adminTags), raw: p}));
177
177
  if (kind === 'collections') {
178
178
  const map = readRegistry().collectionTags;
179
179
  return (await listCollections()).map(c => ({id: c.slug, tags: cleanTagList(map[c.slug]), raw: c}));