maverick-wave-astro 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/components/BaseLayout.astro +4 -0
- package/dist/built-pages.d.ts +21 -0
- package/dist/built-pages.js +67 -0
- package/dist/integration.js +61 -3
- package/dist/root-files.d.ts +9 -0
- package/dist/root-files.js +61 -0
- package/dist/types.d.ts +3 -0
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -26,6 +26,12 @@ The integration sets the CSP-safe build options, the `@` alias, the sitemap,
|
|
|
26
26
|
`satisfies Site` (`src/types.ts`). Colours and fonts stay in the site's
|
|
27
27
|
`main.scss`.
|
|
28
28
|
|
|
29
|
+
After the build it writes `robots.txt` (`site.aiCrawlers`), `llms.txt` and, with
|
|
30
|
+
`site.email`, `/.well-known/security.txt` - only for a site at the root of its
|
|
31
|
+
host, a file in `public/` wins. It warns about titles over 60 characters,
|
|
32
|
+
descriptions outside 140-160, pages without exactly one `<h1>` and images over
|
|
33
|
+
200 KB.
|
|
34
|
+
|
|
29
35
|
## Design
|
|
30
36
|
|
|
31
37
|
`design` in `site.ts` takes the semantic values from `src/design.ts`, e.g.
|
|
@@ -53,6 +53,9 @@ const previewImageAlt =
|
|
|
53
53
|
ogImageAlt ?? (ogImage ? title : (OG_IMAGE.alt ?? title));
|
|
54
54
|
const hasKnownImageSize = !ogImage;
|
|
55
55
|
|
|
56
|
+
// The integration writes llms.txt only for an indexable site at the root.
|
|
57
|
+
const hasLlmsTxt = !site.noIndex && withBase('/') === '/';
|
|
58
|
+
|
|
56
59
|
const robots =
|
|
57
60
|
site.noIndex || noIndex
|
|
58
61
|
? 'noindex, nofollow'
|
|
@@ -89,6 +92,7 @@ const dataAttributes = Object.fromEntries(
|
|
|
89
92
|
))
|
|
90
93
|
}
|
|
91
94
|
<meta name="robots" content={robots} />
|
|
95
|
+
{hasLlmsTxt && <link rel="alternate" type="text/plain" href="/llms.txt" />}
|
|
92
96
|
<meta
|
|
93
97
|
name="format-detection"
|
|
94
98
|
content="telephone=no,email=no,address=no,date=no"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export type Question = {
|
|
2
|
+
question: string;
|
|
3
|
+
answer: string;
|
|
4
|
+
};
|
|
5
|
+
export type BuiltPage = {
|
|
6
|
+
url: string;
|
|
7
|
+
title: string;
|
|
8
|
+
description: string;
|
|
9
|
+
noIndex: boolean;
|
|
10
|
+
headings: number;
|
|
11
|
+
questions: Question[];
|
|
12
|
+
};
|
|
13
|
+
export declare const IMAGE_MAX_BYTES: number;
|
|
14
|
+
export declare const readPage: (html: string) => BuiltPage | undefined;
|
|
15
|
+
export declare const readBuiltPages: (dir: string) => BuiltPage[];
|
|
16
|
+
/** What would cost the page its snippet in the search results. */
|
|
17
|
+
export declare const seoWarnings: (page: BuiltPage) => string[];
|
|
18
|
+
export declare const largeImages: (dir: string) => {
|
|
19
|
+
file: string;
|
|
20
|
+
bytes: number;
|
|
21
|
+
}[];
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
const TITLE_MAX = 60;
|
|
4
|
+
const DESCRIPTION_RANGE = [140, 160];
|
|
5
|
+
export const IMAGE_MAX_BYTES = 200 * 1024;
|
|
6
|
+
const ENTITIES = {
|
|
7
|
+
amp: '&',
|
|
8
|
+
lt: '<',
|
|
9
|
+
gt: '>',
|
|
10
|
+
quot: '"',
|
|
11
|
+
apos: "'",
|
|
12
|
+
};
|
|
13
|
+
const codePoint = (name) => name.startsWith('#x')
|
|
14
|
+
? String.fromCodePoint(parseInt(name.slice(2), 16))
|
|
15
|
+
: String.fromCodePoint(Number(name.slice(1)));
|
|
16
|
+
const decode = (text) => text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, name) => name.startsWith('#') ? codePoint(name) : (ENTITIES[name] ?? entity));
|
|
17
|
+
const JSON_LD = /<script type="application\/ld\+json">([\s\S]*?)<\/script>/g;
|
|
18
|
+
const nodes = (data) => {
|
|
19
|
+
if (Array.isArray(data))
|
|
20
|
+
return data.flatMap(nodes);
|
|
21
|
+
if (!data || typeof data !== 'object')
|
|
22
|
+
return [];
|
|
23
|
+
const node = data;
|
|
24
|
+
return [node, ...nodes(node['@graph'])];
|
|
25
|
+
};
|
|
26
|
+
const questions = (html) => [...html.matchAll(JSON_LD)]
|
|
27
|
+
.flatMap(([, json]) => nodes(JSON.parse(json)))
|
|
28
|
+
.filter(node => node['@type'] === 'FAQPage')
|
|
29
|
+
.flatMap(node => node.mainEntity ?? [])
|
|
30
|
+
.map(entry => ({
|
|
31
|
+
question: String(entry.name),
|
|
32
|
+
answer: String(entry.acceptedAnswer?.text ?? ''),
|
|
33
|
+
}));
|
|
34
|
+
// The head comes from BaseLayout, so a pattern per tag is enough - no parser.
|
|
35
|
+
export const readPage = (html) => {
|
|
36
|
+
const url = /<link rel="canonical" href="([^"]*)"/.exec(html)?.[1];
|
|
37
|
+
const title = /<title>([\s\S]*?)<\/title>/.exec(html)?.[1];
|
|
38
|
+
if (!url || title === undefined)
|
|
39
|
+
return undefined;
|
|
40
|
+
const description = /<meta name="description" content="([^"]*)"/.exec(html)?.[1] ?? '';
|
|
41
|
+
const robots = /<meta name="robots" content="([^"]*)"/.exec(html)?.[1] ?? '';
|
|
42
|
+
return {
|
|
43
|
+
url: decode(url),
|
|
44
|
+
title: decode(title).trim(),
|
|
45
|
+
description: decode(description),
|
|
46
|
+
noIndex: robots.includes('noindex'),
|
|
47
|
+
headings: html.match(/<h1[\s>]/g)?.length ?? 0,
|
|
48
|
+
questions: questions(html),
|
|
49
|
+
};
|
|
50
|
+
};
|
|
51
|
+
const files = (dir, extension) => readdirSync(dir, { recursive: true, encoding: 'utf8' }).filter(file => extension.test(file));
|
|
52
|
+
export const readBuiltPages = (dir) => files(dir, /\.html$/).flatMap(file => readPage(readFileSync(path.join(dir, file), 'utf8')) ?? []);
|
|
53
|
+
/** What would cost the page its snippet in the search results. */
|
|
54
|
+
export const seoWarnings = (page) => {
|
|
55
|
+
const [min, max] = DESCRIPTION_RANGE;
|
|
56
|
+
const length = page.description.length;
|
|
57
|
+
return [
|
|
58
|
+
page.title.length > TITLE_MAX &&
|
|
59
|
+
`title has ${page.title.length} characters, the search shows about ${TITLE_MAX}`,
|
|
60
|
+
(length < min || length > max) &&
|
|
61
|
+
`description has ${length} characters, ${min}-${max} fit the snippet`,
|
|
62
|
+
page.headings !== 1 && `${page.headings} <h1> on the page, one is right`,
|
|
63
|
+
].filter((warning) => typeof warning === 'string');
|
|
64
|
+
};
|
|
65
|
+
export const largeImages = (dir) => files(dir, /\.(jpe?g|png|webp|avif)$/i)
|
|
66
|
+
.map(file => ({ file, bytes: statSync(path.join(dir, file)).size }))
|
|
67
|
+
.filter(image => image.bytes > IMAGE_MAX_BYTES);
|
package/dist/integration.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import sitemap, {} from '@astrojs/sitemap';
|
|
5
|
+
import { IMAGE_MAX_BYTES, largeImages, readBuiltPages, seoWarnings, } from "./built-pages.js";
|
|
5
6
|
import { loadWerkbank } from "./content.js";
|
|
6
7
|
import { designClasses, designCss, designFromSetup, mergeDesigns, validateDesign, } from "./design.js";
|
|
7
8
|
import { occasionEntries } from "./occasions.js";
|
|
9
|
+
import { llmsTxt, robotsTxt, securityTxt } from "./root-files.js";
|
|
8
10
|
import { prepareUploads, removeOriginals } from "./uploads.js";
|
|
9
11
|
const CONFIG_MODULE = 'virtual:maverick-wave-astro/config';
|
|
10
12
|
const WERKBANK_MODULE = 'virtual:maverick-wave-astro/werkbank';
|
|
@@ -48,14 +50,55 @@ export const buildConfig = (options, root, werkbank = loadWerkbank(root)) => {
|
|
|
48
50
|
occasions: occasions ? occasionEntries(occasions) : [],
|
|
49
51
|
};
|
|
50
52
|
};
|
|
53
|
+
// Only a site at the root of its host: /.well-known and robots.txt below a
|
|
54
|
+
// base are never asked for.
|
|
55
|
+
const rootFiles = ({ config: { site }, site: url, base }, sitemap, pages, logger) => {
|
|
56
|
+
if (!url || base.replace(/\/$/, '') !== '')
|
|
57
|
+
return {};
|
|
58
|
+
const origin = new URL(url);
|
|
59
|
+
if (!site.email)
|
|
60
|
+
logger.info('no site.email, so no security.txt');
|
|
61
|
+
return {
|
|
62
|
+
...(site.email && {
|
|
63
|
+
'.well-known/security.txt': securityTxt(origin, site.email, site.lang),
|
|
64
|
+
}),
|
|
65
|
+
...(!site.noIndex && {
|
|
66
|
+
'robots.txt': robotsTxt(origin, { aiCrawlers: site.aiCrawlers, sitemap }),
|
|
67
|
+
'llms.txt': llmsTxt(site.name, site.lang, origin, pages),
|
|
68
|
+
}),
|
|
69
|
+
};
|
|
70
|
+
};
|
|
71
|
+
const writeRootFiles = (out, publicDir, files, logger) => {
|
|
72
|
+
for (const [file, content] of Object.entries(files)) {
|
|
73
|
+
if (existsSync(path.join(publicDir, file))) {
|
|
74
|
+
logger.warn(`public/${file} wins, delete it to get the generated one`);
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
const target = path.join(out, file);
|
|
78
|
+
mkdirSync(path.dirname(target), { recursive: true });
|
|
79
|
+
writeFileSync(target, content);
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
const reportSeo = (out, pages, logger) => {
|
|
83
|
+
for (const page of pages.filter(page => !page.noIndex)) {
|
|
84
|
+
for (const warning of seoWarnings(page)) {
|
|
85
|
+
logger.warn(`${new URL(page.url).pathname}: ${warning}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
for (const { file, bytes } of largeImages(out)) {
|
|
89
|
+
logger.warn(`${file}: ${Math.round(bytes / 1024)} KB, keep images under ${IMAGE_MAX_BYTES / 1024} KB`);
|
|
90
|
+
}
|
|
91
|
+
};
|
|
51
92
|
export default function homepage(options) {
|
|
93
|
+
let build;
|
|
52
94
|
return {
|
|
53
95
|
name: 'maverick-wave-astro',
|
|
54
96
|
hooks: {
|
|
55
97
|
'astro:config:setup': async ({ config: astroConfig, updateConfig, injectRoute, injectScript, addWatchFile, logger, }) => {
|
|
56
98
|
const root = fileURLToPath(astroConfig.root);
|
|
57
99
|
// Before any page renders: Picture reads the sizes from public/.
|
|
58
|
-
const
|
|
100
|
+
const publicDir = fileURLToPath(astroConfig.publicDir);
|
|
101
|
+
const uploads = await prepareUploads(publicDir);
|
|
59
102
|
if (uploads.written > 0)
|
|
60
103
|
logger.info(`${uploads.written} image sizes for uploads written`);
|
|
61
104
|
for (const file of uploads.skipped)
|
|
@@ -65,6 +108,12 @@ export default function homepage(options) {
|
|
|
65
108
|
logger.info(`start content for ${werkbank.fromStart.join(', ')} - nothing from the werkbank`);
|
|
66
109
|
}
|
|
67
110
|
const config = buildConfig(options, root, werkbank);
|
|
111
|
+
build = {
|
|
112
|
+
config,
|
|
113
|
+
site: astroConfig.site,
|
|
114
|
+
base: astroConfig.base,
|
|
115
|
+
publicDir,
|
|
116
|
+
};
|
|
68
117
|
const modules = {
|
|
69
118
|
[CONFIG_MODULE]: config,
|
|
70
119
|
[WERKBANK_MODULE]: {
|
|
@@ -128,7 +177,16 @@ export default function homepage(options) {
|
|
|
128
177
|
});
|
|
129
178
|
}
|
|
130
179
|
},
|
|
131
|
-
'astro:build:done': ({ dir }) =>
|
|
180
|
+
'astro:build:done': ({ dir, logger }) => {
|
|
181
|
+
const out = fileURLToPath(dir);
|
|
182
|
+
removeOriginals(out);
|
|
183
|
+
const pages = readBuiltPages(out);
|
|
184
|
+
if (build) {
|
|
185
|
+
const files = rootFiles(build, options.sitemap !== false, pages, logger);
|
|
186
|
+
writeRootFiles(out, build.publicDir, files, logger);
|
|
187
|
+
}
|
|
188
|
+
reportSeo(out, pages, logger);
|
|
189
|
+
},
|
|
132
190
|
},
|
|
133
191
|
};
|
|
134
192
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { BuiltPage } from './built-pages.ts';
|
|
2
|
+
export type AiCrawlers = 'allow' | 'search-only';
|
|
3
|
+
export declare const robotsTxt: (site: URL, { aiCrawlers, sitemap, }: {
|
|
4
|
+
aiCrawlers?: AiCrawlers;
|
|
5
|
+
sitemap: boolean;
|
|
6
|
+
}) => string;
|
|
7
|
+
export declare const securityTxt: (site: URL, email: string, lang: string, built?: Date) => string;
|
|
8
|
+
/** The indexable pages as llmstxt.org lists them, start page first, then the FAQ. */
|
|
9
|
+
export declare const llmsTxt: (name: string, lang: string, site: URL, pages: readonly BuiltPage[]) => string;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// Collect training data. Google-Extended only covers Gemini - Search and the
|
|
2
|
+
// AI Overviews go through Googlebot, so blocking it costs no visibility.
|
|
3
|
+
const TRAINING_CRAWLERS = ['GPTBot', 'ClaudeBot', 'Google-Extended'];
|
|
4
|
+
// Search index and fetches on a user's request.
|
|
5
|
+
const SEARCH_CRAWLERS = [
|
|
6
|
+
'OAI-SearchBot',
|
|
7
|
+
'ChatGPT-User',
|
|
8
|
+
'Claude-SearchBot',
|
|
9
|
+
'Claude-User',
|
|
10
|
+
'PerplexityBot',
|
|
11
|
+
];
|
|
12
|
+
// RFC 9116 wants Expires less than a year ahead. The date freezes with the build.
|
|
13
|
+
const SECURITY_TXT_DAYS = 360;
|
|
14
|
+
const DAY = 24 * 60 * 60 * 1000;
|
|
15
|
+
const group = (agent, rule) => `User-agent: ${agent}\n${rule}`;
|
|
16
|
+
// A crawler without its own group follows "*" anyway. With 'allow' the groups
|
|
17
|
+
// change nothing, they write the decision down where checkers look for it.
|
|
18
|
+
export const robotsTxt = (site, { aiCrawlers = 'allow', sitemap, }) => {
|
|
19
|
+
const training = aiCrawlers === 'allow' ? 'Allow: /' : 'Disallow: /';
|
|
20
|
+
return `${[
|
|
21
|
+
group('*', 'Allow: /'),
|
|
22
|
+
...TRAINING_CRAWLERS.map(agent => group(agent, training)),
|
|
23
|
+
...SEARCH_CRAWLERS.map(agent => group(agent, 'Allow: /')),
|
|
24
|
+
...(sitemap
|
|
25
|
+
? [`Sitemap: ${new URL('/sitemap-index.xml', site).href}`]
|
|
26
|
+
: []),
|
|
27
|
+
].join('\n\n')}\n`;
|
|
28
|
+
};
|
|
29
|
+
export const securityTxt = (site, email, lang, built = new Date()) => [
|
|
30
|
+
`Contact: mailto:${email}`,
|
|
31
|
+
`Expires: ${new Date(built.getTime() + SECURITY_TXT_DAYS * DAY).toISOString()}`,
|
|
32
|
+
`Preferred-Languages: ${lang}`,
|
|
33
|
+
`Canonical: ${new URL('/.well-known/security.txt', site).href}`,
|
|
34
|
+
'',
|
|
35
|
+
].join('\n');
|
|
36
|
+
const link = (page) => `- [${page.title.replace(/[[\]]/g, '\\$&')}](${page.url})${page.description ? `: ${page.description}` : ''}`;
|
|
37
|
+
/** The indexable pages as llmstxt.org lists them, start page first, then the FAQ. */
|
|
38
|
+
export const llmsTxt = (name, lang, site, pages) => {
|
|
39
|
+
const german = lang.startsWith('de');
|
|
40
|
+
const home = new URL('/', site).href;
|
|
41
|
+
const rank = (page) => (page.url === home ? '' : page.url);
|
|
42
|
+
const listed = pages
|
|
43
|
+
.filter(page => !page.noIndex)
|
|
44
|
+
.sort((a, b) => rank(a).localeCompare(rank(b)));
|
|
45
|
+
const summary = listed.find(page => page.url === home)?.description;
|
|
46
|
+
const questions = [
|
|
47
|
+
...new Map(listed
|
|
48
|
+
.flatMap(page => page.questions)
|
|
49
|
+
.map(entry => [entry.question, entry])).values(),
|
|
50
|
+
];
|
|
51
|
+
return `${[
|
|
52
|
+
`# ${name}`,
|
|
53
|
+
summary && `> ${summary}`,
|
|
54
|
+
`## ${german ? 'Seiten' : 'Pages'}`,
|
|
55
|
+
listed.map(link).join('\n'),
|
|
56
|
+
questions.length > 0 && `## ${german ? 'Häufige Fragen' : 'Questions'}`,
|
|
57
|
+
...questions.map(entry => `### ${entry.question}\n\n${entry.answer}`),
|
|
58
|
+
]
|
|
59
|
+
.filter(Boolean)
|
|
60
|
+
.join('\n\n')}\n`;
|
|
61
|
+
};
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Design } from './design.ts';
|
|
2
2
|
import type { OccasionEntry } from './occasions.ts';
|
|
3
|
+
import type { AiCrawlers } from './root-files.ts';
|
|
3
4
|
export type Site = {
|
|
4
5
|
name: string;
|
|
5
6
|
lang?: string;
|
|
@@ -12,6 +13,8 @@ export type Site = {
|
|
|
12
13
|
hasSvgFavicon?: boolean;
|
|
13
14
|
colorScheme?: 'dark' | 'light';
|
|
14
15
|
noIndex?: boolean;
|
|
16
|
+
email?: string;
|
|
17
|
+
aiCrawlers?: AiCrawlers;
|
|
15
18
|
counter?: boolean;
|
|
16
19
|
themeToggle?: 'system' | 'dark';
|
|
17
20
|
mwScript?: boolean;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "maverick-wave-astro",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.0",
|
|
4
4
|
"description": "Shared frame for static Astro homepages built with MaverickWave: layout, SEO, design variants, schedule, building blocks.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
},
|
|
53
53
|
"peerDependencies": {
|
|
54
54
|
"astro": "^7.2.0",
|
|
55
|
-
"maverick-wave": "^6.
|
|
55
|
+
"maverick-wave": "^6.6.0"
|
|
56
56
|
},
|
|
57
57
|
"dependencies": {
|
|
58
58
|
"@astrojs/sitemap": "^3.7.3",
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"eslint-plugin-astro": "^1.5.0",
|
|
68
68
|
"eslint-plugin-simple-import-sort": "^12.1.1",
|
|
69
69
|
"globals": "^16.5.0",
|
|
70
|
-
"maverick-wave": "^6.
|
|
70
|
+
"maverick-wave": "^6.6.0",
|
|
71
71
|
"prettier": "^3.6.2",
|
|
72
72
|
"prettier-plugin-astro": "^0.14.1",
|
|
73
73
|
"typescript": "^6.0.3",
|