domma-cms 1.60.0 → 1.61.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/CLAUDE.md +1 -1
- package/admin/index.html +7 -1
- package/admin/js/app.js +2 -2
- package/admin/js/lib/ai-assist.js +1 -1
- package/admin/js/templates/ai-assist-builders.html +29 -0
- package/admin/js/templates/docs/usage-chat.html +5 -2
- package/admin/js/views/ai-assist-builders.js +1 -0
- package/admin/js/views/ai-assist-settings.js +1 -1
- package/admin/js/views/index.js +1 -1
- package/admin/js/views/login.js +1 -1
- package/docs/DommaFeatures.md +4 -2
- package/docs/cache.md +24 -0
- package/docs/configuration.md +1 -1
- package/docs/plugin-development.md +26 -0
- package/package.json +1 -1
- package/public/js/effects.js +1 -1
- package/public/js/site.js +1 -1
- package/server/config.js +37 -0
- package/server/routes/api/admin-assistant.js +11 -3
- package/server/routes/api/auth.js +4 -1
- package/server/routes/api/chats.js +3 -1
- package/server/routes/api/components.js +2 -1
- package/server/routes/api/content-api.js +1 -1
- package/server/routes/api/content-public.js +2 -2
- package/server/routes/api/content-writes.js +1 -1
- package/server/routes/api/effects.js +4 -9
- package/server/routes/api/layouts.js +1 -1
- package/server/routes/api/pages.js +6 -5
- package/server/routes/api/search.js +3 -3
- package/server/routes/api/sidebar-counts.js +2 -0
- package/server/routes/api/theme.js +4 -7
- package/server/routes/api/views.js +2 -1
- package/server/routes/public.js +4 -4
- package/server/server.js +14 -7
- package/server/services/aiAssist.js +67 -0
- package/server/services/cache/adminChange.js +28 -0
- package/server/services/chat/ai.js +1 -1
- package/server/services/content.js +73 -10
- package/server/services/effectsSettings.js +25 -0
- package/server/services/health.js +2 -2
- package/server/services/hooks.js +1 -1
- package/server/services/htmlResponse.js +33 -4
- package/server/services/listTags.js +1 -1
- package/server/services/markdown.js +56 -22
- package/server/services/pluginGuides.js +2 -2
- package/server/services/plugins.js +34 -3
- package/server/services/projects.js +39 -4
- package/server/services/publicAssets.js +488 -0
- package/server/services/renderer.js +10 -8
- package/server/services/seo/index.js +3 -3
- package/server/services/sitemap.js +1 -1
- package/server/services/themeSettings.js +11 -0
- package/server/services/versions.js +2 -2
- package/server/services/wpImport/index.js +2 -2
- package/server/templates/page.html +3 -2
|
@@ -20,11 +20,10 @@ import {
|
|
|
20
20
|
requirePermission as defaultRequirePermission
|
|
21
21
|
} from '../../middleware/auth.js';
|
|
22
22
|
import {
|
|
23
|
-
|
|
23
|
+
adminThemePresentation,
|
|
24
24
|
compileOverrideCss,
|
|
25
25
|
getThemeTokenMap,
|
|
26
26
|
listCustomThemes,
|
|
27
|
-
resolveThemeClasses,
|
|
28
27
|
listThemeIds,
|
|
29
28
|
listThemeTokens,
|
|
30
29
|
loadThemeConfig,
|
|
@@ -121,10 +120,7 @@ export async function themeRoutes(fastify, opts = {}) {
|
|
|
121
120
|
if (reply.sent) return reply;
|
|
122
121
|
}
|
|
123
122
|
const id = asked || saved;
|
|
124
|
-
|
|
125
|
-
return {theme: id, dommaTheme, customClass, css: customClass ? compileCustomThemeCss([id]) : '',
|
|
126
|
-
font: {family: cfg.font?.adminFamily || '', stylesheet: cfg.font?.stylesheet || ''},
|
|
127
|
-
sidebar: cfg.adminSidebar !== false};
|
|
123
|
+
return adminThemePresentation(id, cfg);
|
|
128
124
|
});
|
|
129
125
|
|
|
130
126
|
/*
|
|
@@ -146,7 +142,8 @@ export async function themeRoutes(fastify, opts = {}) {
|
|
|
146
142
|
}
|
|
147
143
|
});
|
|
148
144
|
|
|
149
|
-
|
|
145
|
+
// Compiles what it is sent and returns it: not a change (cache/adminChange.js).
|
|
146
|
+
fastify.post('/theme/compile', {...canRead, config: {keepsCache: true}}, async (request, reply) => {
|
|
150
147
|
const candidate = request.body || {};
|
|
151
148
|
const errors = validateThemeConfig(candidate, {remembered: rememberedIds(await loadThemeConfig())});
|
|
152
149
|
if (errors.length) return reply.code(400).send({error: errors.join('; '), validation: errors});
|
|
@@ -181,7 +181,8 @@ export async function viewsRoutes(fastify) {
|
|
|
181
181
|
// user may see (project scope), and it runs as an admin preview - no
|
|
182
182
|
// row-level filter, the same as executing a saved view from the admin.
|
|
183
183
|
// `_` is outside the slug alphabet, so no view can shadow the route.
|
|
184
|
-
|
|
184
|
+
// Runs an unsaved view and returns its rows: not a change (cache/adminChange.js).
|
|
185
|
+
fastify.post('/views/_preview', {...canUpdate, config: {keepsCache: true}}, async (request, reply) => {
|
|
185
186
|
const {source, connection = 'default', stages = [], limit} = request.body || {};
|
|
186
187
|
if (typeof source !== 'string' || !source) return reply.status(400).send({error: 'Choose a source collection'});
|
|
187
188
|
if (!Array.isArray(stages)) return reply.status(400).send({error: 'stages must be an array'});
|
package/server/routes/public.js
CHANGED
|
@@ -17,7 +17,7 @@ import {validSessionCookie, newSessionCookie} from '../services/shortcodeExpiry.
|
|
|
17
17
|
import {touchPreviewLink, verifyPreviewToken} from '../services/previewLinks.js';
|
|
18
18
|
import {viewerRoleKey} from '../services/userRoles.js';
|
|
19
19
|
import {hooks} from '../services/hooks.js';
|
|
20
|
-
import {
|
|
20
|
+
import {readConfig} from '../config.js';
|
|
21
21
|
import * as cache from '../services/cache/index.js';
|
|
22
22
|
|
|
23
23
|
/**
|
|
@@ -34,7 +34,7 @@ import * as cache from '../services/cache/index.js';
|
|
|
34
34
|
* @returns {string} e.g. 'https://example.com' (no trailing slash)
|
|
35
35
|
*/
|
|
36
36
|
export function getBaseUrl(request) {
|
|
37
|
-
const override = (
|
|
37
|
+
const override = (readConfig('site') || {}).baseUrl;
|
|
38
38
|
if (override) return override.toString().trim().replace(/\/+$/, '');
|
|
39
39
|
const host = request.host || request.headers.host || request.hostname;
|
|
40
40
|
return `${request.protocol}://${host}`;
|
|
@@ -283,7 +283,7 @@ export async function publicRoutes(fastify) {
|
|
|
283
283
|
const pageMtime = await getPageMtime(page.urlPath || urlPath);
|
|
284
284
|
const cacheKey = `page:${urlPath}:m${pageMtime}:role${viewerRoleKey(userObj)}:o${baseUrl}`;
|
|
285
285
|
// The footer's blocks (site.json footer.blocks) are on every page: editing one must reach the cached pages.
|
|
286
|
-
const cacheTags = [`page:${urlPath}`, ...(page.cacheTags || []), 'nav', 'site', ...footerBlockCacheTags(
|
|
286
|
+
const cacheTags = [`page:${urlPath}`, ...(page.cacheTags || []), 'nav', 'site', ...footerBlockCacheTags(readConfig('site'))];
|
|
287
287
|
// x-dm-cache says which way this response went - hit, miss, off (the
|
|
288
288
|
// cache is disabled) or bypass (a draft, an expiring or a per-viewer
|
|
289
289
|
// page, never cached) - so a slow page can be read from the browser.
|
|
@@ -294,7 +294,7 @@ export async function publicRoutes(fastify) {
|
|
|
294
294
|
reply.header('x-dm-cache', status);
|
|
295
295
|
hooks.emit('content:pageViewed', {urlPath, title: page.title || ''});
|
|
296
296
|
// cached: this HTML is shared, so its compressed body is kept too (htmlResponse.js).
|
|
297
|
-
return sendHtml(request, reply, html, {cached: true});
|
|
297
|
+
return sendHtml(request, reply, html, {cached: true, key: cacheKey});
|
|
298
298
|
});
|
|
299
299
|
}
|
|
300
300
|
|
package/server/server.js
CHANGED
|
@@ -19,7 +19,7 @@ import path from 'path';
|
|
|
19
19
|
import fs from 'fs/promises';
|
|
20
20
|
import {fileURLToPath} from 'url';
|
|
21
21
|
import {createRequire} from 'module';
|
|
22
|
-
import {config, getConfig} from './config.js';
|
|
22
|
+
import {config, getConfig, readConfig} from './config.js';
|
|
23
23
|
import {getLoadedPlugins, getPluginSettings, recordPluginLoadFailure, registerPlugins} from './services/plugins.js';
|
|
24
24
|
import {notifierFor} from './services/notify.js';
|
|
25
25
|
import {grantAdminToolDefaults, grantPermissionOnce, load as loadRoles, seed as seedRoles} from './services/roles.js';
|
|
@@ -34,6 +34,7 @@ import {parseTrustProxy} from './services/network.js';
|
|
|
34
34
|
import {requestSerializer} from './services/request-log.js';
|
|
35
35
|
import {rateLimitOptions} from './services/rate-limit.js';
|
|
36
36
|
import {registerAdminAssets} from './services/adminAssets.js';
|
|
37
|
+
import {registerPublicAssets} from './services/publicAssets.js';
|
|
37
38
|
import {isActiveType, servedType} from './services/mediaTypes.js';
|
|
38
39
|
import {connectSources, frameSources, imageSources} from './services/embedCsp.js';
|
|
39
40
|
import {cspSources as gaSources} from './services/googleAnalytics.js';
|
|
@@ -131,7 +132,7 @@ await app.register(helmet, {
|
|
|
131
132
|
scriptSrc: ["'self'", "'unsafe-inline'", "'wasm-unsafe-eval'", 'https://cdn.jsdelivr.net', gaSources('script')],
|
|
132
133
|
styleSrc: ["'self'", "'unsafe-inline'", 'https://cdn.jsdelivr.net', 'https://fonts.googleapis.com'],
|
|
133
134
|
// config/server.json csp.imgHosts: other sites a page may show images from (services/embedCsp.js).
|
|
134
|
-
imgSrc: [() => imageSources(
|
|
135
|
+
imgSrc: [() => imageSources(readConfig('server').csp?.imgHosts).join(' '), gaSources('img')],
|
|
135
136
|
fontSrc: ["'self'", 'data:', 'https://fonts.gstatic.com'],
|
|
136
137
|
// config/server.json csp.connectHosts: other sites a page may read from (services/embedCsp.js).
|
|
137
138
|
connectSrc: [...connectSources(serverConfig.csp?.connectHosts), gaSources('connect')],
|
|
@@ -399,11 +400,11 @@ try {
|
|
|
399
400
|
* tag. Runs on onResponse, after the reply has gone, so it costs the caller
|
|
400
401
|
* nothing - and a memory-driver clear is two Map resets.
|
|
401
402
|
*/
|
|
402
|
-
|
|
403
|
+
// A route that changes nothing a page could show opts out with `config: {keepsCache: true}`
|
|
404
|
+
// (services/cache/adminChange.js) - the admin's 30-second chat check-in was emptying the cache.
|
|
405
|
+
const {isCacheClearingChange} = await import('./services/cache/adminChange.js');
|
|
403
406
|
app.addHook('onResponse', async (request, reply) => {
|
|
404
|
-
if (!
|
|
405
|
-
if (reply.statusCode >= 400) return;
|
|
406
|
-
if (!request.user) return;
|
|
407
|
+
if (!isCacheClearingChange(request, reply)) return;
|
|
407
408
|
try {
|
|
408
409
|
await cache.clear();
|
|
409
410
|
} catch (err) {
|
|
@@ -442,6 +443,10 @@ await app.register(staticPlugin, {
|
|
|
442
443
|
}
|
|
443
444
|
});
|
|
444
445
|
|
|
446
|
+
// Fingerprinted public assets are cached for good, and the site's custom CSS is
|
|
447
|
+
// served as a file - services/publicAssets.js.
|
|
448
|
+
const publicAssets = registerPublicAssets(app, {root: ROOT, dommaDist: DOMMA_DIST, pluginsDir});
|
|
449
|
+
|
|
445
450
|
// Serve plugin admin/ and public/ subdirs only - block plugin.js, config.js, data/, etc.
|
|
446
451
|
await app.register(async function pluginStaticScope(instance) {
|
|
447
452
|
instance.addHook('onRequest', (request, reply, done) => {
|
|
@@ -784,13 +789,15 @@ await registerPlugins(app);
|
|
|
784
789
|
const {isAiAssist} = await import('./services/aiAssist.js');
|
|
785
790
|
if (isAiAssist()) {
|
|
786
791
|
const url = isToolEnabled('launchers') ? '#/ai-assist/chats' : '#/ai-assist/settings';
|
|
787
|
-
|
|
792
|
+
// Under Data by default; an admin's own placement in the menu still wins.
|
|
793
|
+
registerSidebarItem({folder: 'data', item: {source: 'Built in', text: 'AI Assist', url, icon: 'sparkles', permission: isToolEnabled('launchers') ? 'chats' : 'settings',
|
|
788
794
|
...(isToolEnabled('launchers') && {countUrl: '/api/chats/_count'})}});
|
|
789
795
|
hideSidebarUrl('#/launchers/chats', url);
|
|
790
796
|
}
|
|
791
797
|
}
|
|
792
798
|
// After the plugins' onRequest hooks - see serve() in services/adminAssets.js.
|
|
793
799
|
adminAssets.serve();
|
|
800
|
+
publicAssets.serve();
|
|
794
801
|
|
|
795
802
|
// ---------------------------------------------------------------------------
|
|
796
803
|
// Public Plugin Routes (root-level, before catch-all)
|
|
@@ -50,7 +50,74 @@ export function chatsHome() {
|
|
|
50
50
|
return isAiAssist() ? AI_ASSIST_CHATS : LAUNCHER_CHATS;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
// Builders: AI chats of other Tools, listed on AI Assist's Builders tab
|
|
55
|
+
// ---------------------------------------------------------------------------
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* plugin name -> its builder entry. A plugin declares one in plugin.json:
|
|
59
|
+
*
|
|
60
|
+
* "admin": {"aiAssist": {"label": "Site Builder", "url": "#/plugins/ai-site-builder",
|
|
61
|
+
* "list": "/api/plugins/ai-site-builder/conversations", "permission": "ai-builder",
|
|
62
|
+
* "character": "dot"}}
|
|
63
|
+
*
|
|
64
|
+
* `url` must be the plugin's own admin screen and `list` its own API: AI Assist only
|
|
65
|
+
* links to the plugin and reads its list with the viewer's own token, so the plugin's
|
|
66
|
+
* routes still decide what each person sees. Registered by the loader for enabled plugins.
|
|
67
|
+
*/
|
|
68
|
+
const builders = new Map();
|
|
69
|
+
const SLUG = /^[a-z0-9][a-z0-9-]{0,63}$/;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* What is wrong with a plugin's `admin.aiAssist` block, or ''.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} plugin - the manifest name
|
|
75
|
+
* @param {object} block
|
|
76
|
+
*/
|
|
77
|
+
export function aiAssistBuilderProblem(plugin, block) {
|
|
78
|
+
if (!SLUG.test(String(plugin || ''))) return 'invalid plugin name';
|
|
79
|
+
if (!block || typeof block !== 'object' || Array.isArray(block)) return 'admin.aiAssist must be an object';
|
|
80
|
+
const own = (re) => new RegExp(re.replace('<p>', plugin.replace(/-/g, '\\-')));
|
|
81
|
+
if (typeof block.label !== 'string' || !block.label.trim() || block.label.length > 40) return 'label must be text of up to 40 characters';
|
|
82
|
+
if (typeof block.url !== 'string' || !own('^#/plugins/<p>(/[a-z0-9-]+)*$').test(block.url)) return `url must be this plugin's own screen (#/plugins/${plugin}...)`;
|
|
83
|
+
if (typeof block.list !== 'string' || !own('^/api/plugins/<p>(/[a-z0-9-]+)+$').test(block.list)) return `list must be this plugin's own API (/api/plugins/${plugin}/...)`;
|
|
84
|
+
if (typeof block.permission !== 'string' || !/^[a-z0-9][a-z0-9.-]{0,79}$/.test(block.permission)) return 'permission is required';
|
|
85
|
+
if (block.character !== undefined && (typeof block.character !== 'string' || !SLUG.test(block.character))) return 'character must be a character id';
|
|
86
|
+
return '';
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Register a loaded plugin's builder entry; returns the problem ('' when registered). */
|
|
90
|
+
export function registerAiAssistBuilder(plugin, block) {
|
|
91
|
+
const problem = aiAssistBuilderProblem(plugin, block);
|
|
92
|
+
if (problem) return problem;
|
|
93
|
+
builders.set(plugin, {plugin, label: block.label.trim(), url: block.url, list: block.list, permission: block.permission,
|
|
94
|
+
...(block.character ? {character: block.character} : {})});
|
|
95
|
+
return '';
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** The plugin was switched off: its chats leave the Builders tab at once, not at the next restart. */
|
|
99
|
+
export function unregisterAiAssistBuilder(plugin) {
|
|
100
|
+
return builders.delete(String(plugin || ''));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The builders this viewer may open: they hold the entry's permission (the resource or
|
|
105
|
+
* any action of it), and a confined user's admin scope covers its screen.
|
|
106
|
+
*
|
|
107
|
+
* @param {{permissions: string[], adminScope?: {allow?: string[]}|null}} access - services/adminAccess.js adminAccessFor()
|
|
108
|
+
* @param {(prefix: string, path: string) => boolean} covers - services/pluginScope.js covers()
|
|
109
|
+
*/
|
|
110
|
+
export function aiAssistBuildersFor(access, covers) {
|
|
111
|
+
const perms = access?.permissions || [];
|
|
112
|
+
const holds = (p) => perms.includes(p) || perms.some(x => x.startsWith(`${p}.`)) || perms.includes(p.split('.')[0]);
|
|
113
|
+
const scope = access?.adminScope;
|
|
114
|
+
const inScope = (url) => !scope || (scope.allow || []).some(prefix => covers(prefix, url.slice(1)));
|
|
115
|
+
return [...builders.values()].filter(b => holds(b.permission) && inScope(b.url))
|
|
116
|
+
.map(({plugin, label, url, list, character}) => ({plugin, label, url, list, ...(character ? {character} : {})}));
|
|
117
|
+
}
|
|
118
|
+
|
|
53
119
|
/** Tests only: forget every owner. */
|
|
54
120
|
export function _resetAiAssistForTests() {
|
|
55
121
|
owners.clear();
|
|
122
|
+
builders.clear();
|
|
56
123
|
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does this finished request count as an admin change that clears the
|
|
3
|
+
* response cache? (The safety net in server.js - see the comment there.)
|
|
4
|
+
*
|
|
5
|
+
* A signed-in, successful POST/PUT/PATCH/DELETE does - unless its route says
|
|
6
|
+
* it changes nothing a page could show: `config: {keepsCache: true}`. The
|
|
7
|
+
* admin's "available for chats" check-in is one, sent every 30 seconds from
|
|
8
|
+
* every admin screen; counted as a change, one signed-in person emptied the
|
|
9
|
+
* whole cache twice a minute and no page was ever served from it.
|
|
10
|
+
*
|
|
11
|
+
* Only for a route whose effect lives outside every rendered page. When in
|
|
12
|
+
* doubt leave it off: a cache cleared too often is slow, a cache not cleared
|
|
13
|
+
* is wrong.
|
|
14
|
+
*/
|
|
15
|
+
const CLEARING_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {{method: string, user?: object, routeOptions?: {config?: {keepsCache?: boolean}}}} request
|
|
19
|
+
* @param {{statusCode: number}} reply
|
|
20
|
+
* @returns {boolean}
|
|
21
|
+
*/
|
|
22
|
+
export function isCacheClearingChange(request, reply) {
|
|
23
|
+
if (!CLEARING_METHODS.has(request.method)) return false;
|
|
24
|
+
if (reply.statusCode >= 400) return false;
|
|
25
|
+
if (!request.user) return false;
|
|
26
|
+
if (request.routeOptions?.config?.keepsCache === true) return false;
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
@@ -22,7 +22,7 @@ const REGISTRY = Symbol.for('domma.launcher-chat');
|
|
|
22
22
|
/** What a visitor will wait for an answer. */
|
|
23
23
|
const ASK_MS = 25_000;
|
|
24
24
|
/** Knowledge and chat sent at most: a reply is a few sentences, the prompt need not be a book. */
|
|
25
|
-
export const KNOWLEDGE_CHARS =
|
|
25
|
+
export const KNOWLEDGE_CHARS = 48_000;
|
|
26
26
|
export const HISTORY_MESSAGES = 10;
|
|
27
27
|
/** After the Manager says no (no licence, the month's answers used), ask again only after this. */
|
|
28
28
|
const BACKOFF_MS = {403: 10 * 60_000, 429: 30 * 60_000, other: 60_000};
|
|
@@ -35,6 +35,9 @@ const MEDIA_DIR = config.content.mediaDir;
|
|
|
35
35
|
// the hooks do not know about. Entries are cloned in and out, so a caller
|
|
36
36
|
// that edits the object it was handed edits its own copy.
|
|
37
37
|
const PAGE_MEMO_MAX = 500;
|
|
38
|
+
/** listPages() raises the limit to the number of pages, up to this. */
|
|
39
|
+
const PAGE_MEMO_CEILING = 5000;
|
|
40
|
+
let _pageMemoMax = PAGE_MEMO_MAX;
|
|
38
41
|
const PAGE_MEMO_TTL_MS = 5 * 60 * 1000;
|
|
39
42
|
/** @type {Map<string, {mtimeMs: number, size: number, at: number, page: object}>} */
|
|
40
43
|
const _pageMemo = new Map();
|
|
@@ -43,11 +46,13 @@ cache.onInvalidate(() => _pageMemo.clear());
|
|
|
43
46
|
|
|
44
47
|
/** For the cache status route and tests: how the memo is doing. */
|
|
45
48
|
export function pageMemoStats() {
|
|
46
|
-
return {size: _pageMemo.size, max:
|
|
49
|
+
return {size: _pageMemo.size, max: _pageMemoMax, hits: _pageMemoStats.hits, misses: _pageMemoStats.misses};
|
|
47
50
|
}
|
|
48
51
|
|
|
49
52
|
export function _resetPageMemoForTests() {
|
|
50
53
|
_pageMemo.clear();
|
|
54
|
+
_summaryMemo.clear();
|
|
55
|
+
_pageMemoMax = PAGE_MEMO_MAX;
|
|
51
56
|
_pageMemoStats.hits = 0;
|
|
52
57
|
_pageMemoStats.misses = 0;
|
|
53
58
|
}
|
|
@@ -69,7 +74,7 @@ async function readPageMemo(filePath) {
|
|
|
69
74
|
} catch {
|
|
70
75
|
return page; // something in it cannot be cloned: hand it over unmemoised
|
|
71
76
|
}
|
|
72
|
-
while (_pageMemo.size >
|
|
77
|
+
while (_pageMemo.size > _pageMemoMax) _pageMemo.delete(_pageMemo.keys().next().value);
|
|
73
78
|
return page;
|
|
74
79
|
}
|
|
75
80
|
|
|
@@ -77,21 +82,80 @@ async function readPageMemo(filePath) {
|
|
|
77
82
|
// Pages
|
|
78
83
|
// ---------------------------------------------------------------------------
|
|
79
84
|
|
|
85
|
+
// ---------------------------------------------------------------------------
|
|
86
|
+
// Page summaries (front matter + source, nothing rendered)
|
|
87
|
+
// ---------------------------------------------------------------------------
|
|
88
|
+
//
|
|
89
|
+
// A page's front matter and Markdown source depend on its file and nothing
|
|
90
|
+
// else, so they are kept per file and checked against the file's mtime and
|
|
91
|
+
// size on every read - no TTL, and correct with the response cache on or off.
|
|
92
|
+
// Emptied with the response cache all the same, so two saves inside one mtime
|
|
93
|
+
// tick cannot leave the first one's values behind. Cloned out, so a caller
|
|
94
|
+
// that edits a row edits its own copy.
|
|
95
|
+
/** @type {Map<string, {mtimeMs: number, size: number, page: object}>} */
|
|
96
|
+
const _summaryMemo = new Map();
|
|
97
|
+
cache.onInvalidate(() => _summaryMemo.clear());
|
|
98
|
+
|
|
99
|
+
async function readPageSummary(filePath) {
|
|
100
|
+
const st = await fs.stat(filePath);
|
|
101
|
+
const kept = _summaryMemo.get(filePath);
|
|
102
|
+
if (kept && kept.mtimeMs === st.mtimeMs && kept.size === st.size) return structuredClone(kept.page);
|
|
103
|
+
const raw = await fs.readFile(filePath, 'utf8');
|
|
104
|
+
// Options passed on purpose: without them gray-matter keeps every string it
|
|
105
|
+
// has ever parsed (and hands the same `data` object to every caller).
|
|
106
|
+
const {data, content} = matter(raw, {});
|
|
107
|
+
const page = {...data, urlPath: filePathToUrlPath(filePath), content};
|
|
108
|
+
try {
|
|
109
|
+
_summaryMemo.set(filePath, {mtimeMs: st.mtimeMs, size: st.size, page: structuredClone(page)});
|
|
110
|
+
} catch { /* something in the front matter cannot be cloned: read it each time */ }
|
|
111
|
+
return page;
|
|
112
|
+
}
|
|
113
|
+
|
|
80
114
|
/**
|
|
81
115
|
* Recursively list all .md files under PAGES_DIR.
|
|
82
|
-
* Returns
|
|
116
|
+
* Returns page objects sorted by sortOrder then title.
|
|
117
|
+
*
|
|
118
|
+
* By default every page is parsed in full, as getPage() does - shortcodes,
|
|
119
|
+
* collections, sanitising - which is tens of milliseconds a page and seconds
|
|
120
|
+
* for a few hundred. A caller that reads only the front matter (title, status,
|
|
121
|
+
* layout, tags, seo, ...) or the Markdown source (`content`) asks for
|
|
122
|
+
* `{render: false}` and gets exactly those plus `urlPath` and
|
|
123
|
+
* `resolvedProject`: no `html`, `usedComponents`, `cacheTags`, `perViewer` or
|
|
124
|
+
* `hasExpiry`. Lists, badges, search and the sitemap all do.
|
|
125
|
+
*
|
|
126
|
+
* The rendered rows come through the parsed-page memo while the response
|
|
127
|
+
* cache is on, so a second listing does not parse every page again.
|
|
83
128
|
*
|
|
129
|
+
* @param {{render?: boolean}} [opts]
|
|
84
130
|
* @returns {Promise<object[]>}
|
|
85
131
|
*/
|
|
86
|
-
export async function listPages() {
|
|
132
|
+
export async function listPages({render = true} = {}) {
|
|
87
133
|
const files = await collectMdFiles(PAGES_DIR);
|
|
88
|
-
|
|
134
|
+
let pages;
|
|
135
|
+
if (!render) {
|
|
136
|
+
pages = await Promise.all(files.map(filePath => readPageSummary(filePath)));
|
|
137
|
+
// Forget pages that are gone, so the memo holds the site's pages and no more.
|
|
138
|
+
if (_summaryMemo.size > files.length) {
|
|
139
|
+
const live = new Set(files);
|
|
140
|
+
for (const key of _summaryMemo.keys()) if (!live.has(key)) _summaryMemo.delete(key);
|
|
141
|
+
}
|
|
142
|
+
} else if (cache.isEnabled()) {
|
|
143
|
+
// Every page must fit at once, or a listing evicts what it is about to read.
|
|
144
|
+
_pageMemoMax = Math.max(_pageMemoMax, Math.min(PAGE_MEMO_CEILING, files.length + 50));
|
|
145
|
+
pages = await Promise.all(files.map(filePath => readPageMemo(filePath)));
|
|
146
|
+
} else {
|
|
147
|
+
pages = await Promise.all(files.map(filePath => readPageFile(filePath)));
|
|
148
|
+
}
|
|
89
149
|
pages.sort((a, b) => (a.sortOrder ?? 99) - (b.sortOrder ?? 99) || a.title.localeCompare(b.title));
|
|
90
|
-
|
|
150
|
+
// One read of the projects for the whole list, not one per page.
|
|
151
|
+
const {listProjects, projectForPage} = await import('./projects.js');
|
|
152
|
+
let projects = null;
|
|
91
153
|
for (const page of pages) {
|
|
92
154
|
const url = page.url || page.urlPath || '/';
|
|
93
155
|
const explicit = (page.project === undefined) ? undefined : page.project;
|
|
94
|
-
page.resolvedProject =
|
|
156
|
+
if (typeof explicit === 'string' && explicit) { page.resolvedProject = explicit; continue; }
|
|
157
|
+
projects ??= await listProjects();
|
|
158
|
+
page.resolvedProject = projectForPage(projects, url, explicit);
|
|
95
159
|
}
|
|
96
160
|
return pages;
|
|
97
161
|
}
|
|
@@ -327,9 +391,8 @@ export async function setPageFrontmatter(urlPath, patch) {
|
|
|
327
391
|
throw err;
|
|
328
392
|
}
|
|
329
393
|
const raw = await fs.readFile(filePath, 'utf8');
|
|
330
|
-
// gray-matter
|
|
331
|
-
|
|
332
|
-
const parsed = matter(raw);
|
|
394
|
+
// Options passed so gray-matter does not keep the string (see readPageSummary).
|
|
395
|
+
const parsed = matter(raw, {});
|
|
333
396
|
const data = {...parsed.data};
|
|
334
397
|
for (const [k, v] of Object.entries(patch || {})) {
|
|
335
398
|
if (v === undefined) delete data[k];
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The scroll-reveal defaults every public page's effects read
|
|
3
|
+
* (config/effects.json over the built-in defaults).
|
|
4
|
+
*
|
|
5
|
+
* Its own module so both the route that serves them and the renderer - which
|
|
6
|
+
* writes them into the page, saving every page view a request before its
|
|
7
|
+
* effects can start - read the same thing.
|
|
8
|
+
*/
|
|
9
|
+
import {getConfig} from '../config.js';
|
|
10
|
+
|
|
11
|
+
export const EFFECTS_DEFAULTS = {respectMotion: true, defaultDuration: 600, defaultAnimation: 'fade', defaultThreshold: 0.1};
|
|
12
|
+
|
|
13
|
+
/** @returns {object} the defaults with config/effects.json over them */
|
|
14
|
+
export function loadEffectsSettings() {
|
|
15
|
+
try {
|
|
16
|
+
return {...EFFECTS_DEFAULTS, ...(getConfig('effects') || {})};
|
|
17
|
+
} catch {
|
|
18
|
+
return {...EFFECTS_DEFAULTS};
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** `window.__CMS_EFFECTS__ = {...};` for a page's inline script (public/js/effects.js reads it). */
|
|
23
|
+
export function effectsSettingsScript() {
|
|
24
|
+
return `window.__CMS_EFFECTS__ = ${JSON.stringify(loadEffectsSettings()).replace(/</g, '\\u003c')};`;
|
|
25
|
+
}
|
|
@@ -125,7 +125,7 @@ async function checkContentDisk() {
|
|
|
125
125
|
}
|
|
126
126
|
|
|
127
127
|
async function checkStalePages() {
|
|
128
|
-
const pages = await listPages().catch(() => []);
|
|
128
|
+
const pages = await listPages({render: false}).catch(() => []);
|
|
129
129
|
const cutoff = Date.now() - STALE_PAGE_DAYS * 24 * 60 * 60 * 1000;
|
|
130
130
|
const stale = pages.filter(p => {
|
|
131
131
|
const updated = p.updatedAt ? new Date(p.updatedAt).getTime() : 0;
|
|
@@ -141,7 +141,7 @@ async function checkStalePages() {
|
|
|
141
141
|
}
|
|
142
142
|
|
|
143
143
|
async function checkMissingMeta() {
|
|
144
|
-
const pages = await listPages().catch(() => []);
|
|
144
|
+
const pages = await listPages({render: false}).catch(() => []);
|
|
145
145
|
const missing = pages.filter(p => !p.description || String(p.description).trim() === '');
|
|
146
146
|
return {
|
|
147
147
|
id: 'missing-meta',
|
package/server/services/hooks.js
CHANGED
|
@@ -99,7 +99,7 @@ const _registeredSidebarItems = [];
|
|
|
99
99
|
*/
|
|
100
100
|
const TOOL_GROUPS = new Map(Object.entries({
|
|
101
101
|
Productivity: ['#/notes', '#/todo', '#/calculator', '#/plugins/calendar', '#/plugins/waypoint-pro', '#/plugins/sheets'],
|
|
102
|
-
Communication: ['#/
|
|
102
|
+
Communication: ['#/contacts', '#/plugins/contacts-pro', '#/plugins/mail-reader', '#/plugins/email-pro', '#/plugins/messages'],
|
|
103
103
|
Marketing: ['#/analytics', '#/seo', '#/plugins/analytics-pro/dashboard', '#/plugins/social', '#/plugins/feedback', '#/plugins/surveys', '#/plugins/campaigns-pro', '#/plugins/ecards'],
|
|
104
104
|
Publishing: ['#/plugins/blog', '#/plugins/blog-pro', '#/plugins/docs', '#/plugins/theme-roller', '#/plugins/wiki', '#/plugins/wiki-pro', '#/plugins/scraper', '#/plugins/scraper-pro'],
|
|
105
105
|
Business: ['#/plugins/shopping-cart', '#/plugins/booking', '#/plugins/booking-pro', '#/plugins/invoice', '#/job-board', '#/plugins/garage', '#/plugins/takeaway', '#/plugins/support', '#/plugins/onboard-pro'],
|
|
@@ -96,6 +96,34 @@ export function htmlHash(html) {
|
|
|
96
96
|
return createHash('sha1').update(html).digest('base64');
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
// A page served from the response cache is the SAME string on every hit, and
|
|
100
|
+
// hashing it (sha1 over the whole document) was half of what a hit cost. The
|
|
101
|
+
// hash is kept per cache key beside the string it was made from, and used only
|
|
102
|
+
// while the caller hands in that very string - a page rendered again is a new
|
|
103
|
+
// string, fails the comparison (identity first, so a hit compares nothing) and
|
|
104
|
+
// is hashed afresh. So a kept hash can never describe other HTML. Dropped
|
|
105
|
+
// whenever anything leaves the response cache, so it does not hold a page the
|
|
106
|
+
// cache has let go.
|
|
107
|
+
const HASH_MEMO_MAX = 2000;
|
|
108
|
+
/** @type {Map<string, {html: string, hash: string}>} */
|
|
109
|
+
const _hashMemo = new Map();
|
|
110
|
+
cache.onInvalidate(() => _hashMemo.clear());
|
|
111
|
+
|
|
112
|
+
function memoisedHash(key, html) {
|
|
113
|
+
const kept = _hashMemo.get(key);
|
|
114
|
+
if (kept && kept.html === html) return kept.hash;
|
|
115
|
+
const hash = htmlHash(html);
|
|
116
|
+
_hashMemo.delete(key);
|
|
117
|
+
_hashMemo.set(key, {html, hash});
|
|
118
|
+
while (_hashMemo.size > HASH_MEMO_MAX) _hashMemo.delete(_hashMemo.keys().next().value);
|
|
119
|
+
return hash;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** For tests: how many hashes are kept. */
|
|
123
|
+
export function _hashMemoSize() {
|
|
124
|
+
return _hashMemo.size;
|
|
125
|
+
}
|
|
126
|
+
|
|
99
127
|
/**
|
|
100
128
|
* Send a rendered page: cache-control, an ETag, a 304 when the browser
|
|
101
129
|
* already has it, and - for a response that came from the response cache -
|
|
@@ -114,11 +142,12 @@ export function htmlHash(html) {
|
|
|
114
142
|
* @param {import('fastify').FastifyRequest} request
|
|
115
143
|
* @param {import('fastify').FastifyReply} reply
|
|
116
144
|
* @param {string} html
|
|
117
|
-
* @param {{cached?: boolean}} [opts] - cached: the HTML came from the response cache, so its
|
|
118
|
-
* compressed form may be kept too
|
|
145
|
+
* @param {{cached?: boolean, key?: string}} [opts] - cached: the HTML came from the response cache, so its
|
|
146
|
+
* compressed form may be kept too; key: the response-cache key it came from, so its hash is
|
|
147
|
+
* kept as well (only with `cached`)
|
|
119
148
|
*/
|
|
120
|
-
export async function sendHtml(request, reply, html, {cached = false} = {}) {
|
|
121
|
-
const hash = htmlHash(html);
|
|
149
|
+
export async function sendHtml(request, reply, html, {cached = false, key = ''} = {}) {
|
|
150
|
+
const hash = cached && key && cache.isEnabled() ? memoisedHash(key, html) : htmlHash(html);
|
|
122
151
|
const encoding = cached && Buffer.byteLength(html) >= COMPRESS_THRESHOLD
|
|
123
152
|
? pickEncoding(request.headers['accept-encoding'])
|
|
124
153
|
: null;
|
|
@@ -173,7 +173,7 @@ function toolTagFor(kind, raw, names) {
|
|
|
173
173
|
|
|
174
174
|
/** The items of a kind as `{id, tags, raw}`, through the same listers the routes use. */
|
|
175
175
|
async function items(kind) {
|
|
176
|
-
if (kind === 'pages') return (await listPages()).map(p => ({id: p.urlPath, tags: cleanTagList(p.adminTags), raw: p}));
|
|
176
|
+
if (kind === 'pages') return (await listPages({render: false})).map(p => ({id: p.urlPath, tags: cleanTagList(p.adminTags), raw: p}));
|
|
177
177
|
if (kind === 'collections') {
|
|
178
178
|
const map = readRegistry().collectionTags;
|
|
179
179
|
return (await listCollections()).map(c => ({id: c.slug, tags: cleanTagList(map[c.slug]), raw: c}));
|