maverick-wave-astro 1.8.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -53,6 +53,9 @@ const previewImageAlt =
53
53
  ogImageAlt ?? (ogImage ? title : (OG_IMAGE.alt ?? title));
54
54
  const hasKnownImageSize = !ogImage;
55
55
 
56
+ // The integration writes llms.txt only for an indexable site at the root.
57
+ const hasLlmsTxt = !site.noIndex && withBase('/') === '/';
58
+
56
59
  const robots =
57
60
  site.noIndex || noIndex
58
61
  ? 'noindex, nofollow'
@@ -89,6 +92,7 @@ const dataAttributes = Object.fromEntries(
89
92
  ))
90
93
  }
91
94
  <meta name="robots" content={robots} />
95
+ {hasLlmsTxt && <link rel="alternate" type="text/plain" href="/llms.txt" />}
92
96
  <meta
93
97
  name="format-detection"
94
98
  content="telephone=no,email=no,address=no,date=no"
@@ -0,0 +1,21 @@
1
+ export type Question = {
2
+ question: string;
3
+ answer: string;
4
+ };
5
+ export type BuiltPage = {
6
+ url: string;
7
+ title: string;
8
+ description: string;
9
+ noIndex: boolean;
10
+ headings: number;
11
+ questions: Question[];
12
+ };
13
+ export declare const IMAGE_MAX_BYTES: number;
14
+ export declare const readPage: (html: string) => BuiltPage | undefined;
15
+ export declare const readBuiltPages: (dir: string) => BuiltPage[];
16
+ /** What would cost the page its snippet in the search results. */
17
+ export declare const seoWarnings: (page: BuiltPage) => string[];
18
+ export declare const largeImages: (dir: string) => {
19
+ file: string;
20
+ bytes: number;
21
+ }[];
@@ -0,0 +1,67 @@
1
+ import { readdirSync, readFileSync, statSync } from 'node:fs';
2
+ import path from 'node:path';
3
+ const TITLE_MAX = 60;
4
+ const DESCRIPTION_RANGE = [140, 160];
5
+ export const IMAGE_MAX_BYTES = 200 * 1024;
6
+ const ENTITIES = {
7
+ amp: '&',
8
+ lt: '<',
9
+ gt: '>',
10
+ quot: '"',
11
+ apos: "'",
12
+ };
13
+ const codePoint = (name) => name.startsWith('#x')
14
+ ? String.fromCodePoint(parseInt(name.slice(2), 16))
15
+ : String.fromCodePoint(Number(name.slice(1)));
16
+ const decode = (text) => text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, name) => name.startsWith('#') ? codePoint(name) : (ENTITIES[name] ?? entity));
17
+ const JSON_LD = /<script type="application\/ld\+json">([\s\S]*?)<\/script>/g;
18
+ const nodes = (data) => {
19
+ if (Array.isArray(data))
20
+ return data.flatMap(nodes);
21
+ if (!data || typeof data !== 'object')
22
+ return [];
23
+ const node = data;
24
+ return [node, ...nodes(node['@graph'])];
25
+ };
26
+ const questions = (html) => [...html.matchAll(JSON_LD)]
27
+ .flatMap(([, json]) => nodes(JSON.parse(json)))
28
+ .filter(node => node['@type'] === 'FAQPage')
29
+ .flatMap(node => node.mainEntity ?? [])
30
+ .map(entry => ({
31
+ question: String(entry.name),
32
+ answer: String(entry.acceptedAnswer?.text ?? ''),
33
+ }));
34
+ // The head comes from BaseLayout, so a pattern per tag is enough - no parser.
35
+ export const readPage = (html) => {
36
+ const url = /<link rel="canonical" href="([^"]*)"/.exec(html)?.[1];
37
+ const title = /<title>([\s\S]*?)<\/title>/.exec(html)?.[1];
38
+ if (!url || title === undefined)
39
+ return undefined;
40
+ const description = /<meta name="description" content="([^"]*)"/.exec(html)?.[1] ?? '';
41
+ const robots = /<meta name="robots" content="([^"]*)"/.exec(html)?.[1] ?? '';
42
+ return {
43
+ url: decode(url),
44
+ title: decode(title).trim(),
45
+ description: decode(description),
46
+ noIndex: robots.includes('noindex'),
47
+ headings: html.match(/<h1[\s>]/g)?.length ?? 0,
48
+ questions: questions(html),
49
+ };
50
+ };
51
+ const files = (dir, extension) => readdirSync(dir, { recursive: true, encoding: 'utf8' }).filter(file => extension.test(file));
52
+ export const readBuiltPages = (dir) => files(dir, /\.html$/).flatMap(file => readPage(readFileSync(path.join(dir, file), 'utf8')) ?? []);
53
+ /** What would cost the page its snippet in the search results. */
54
+ export const seoWarnings = (page) => {
55
+ const [min, max] = DESCRIPTION_RANGE;
56
+ const length = page.description.length;
57
+ return [
58
+ page.title.length > TITLE_MAX &&
59
+ `title has ${page.title.length} characters, the search shows about ${TITLE_MAX}`,
60
+ (length < min || length > max) &&
61
+ `description has ${length} characters, ${min}-${max} fit the snippet`,
62
+ page.headings !== 1 && `${page.headings} <h1> on the page, one is right`,
63
+ ].filter((warning) => typeof warning === 'string');
64
+ };
65
+ export const largeImages = (dir) => files(dir, /\.(jpe?g|png|webp|avif)$/i)
66
+ .map(file => ({ file, bytes: statSync(path.join(dir, file)).size }))
67
+ .filter(image => image.bytes > IMAGE_MAX_BYTES);
@@ -1,10 +1,12 @@
1
- import { existsSync, readFileSync } from 'node:fs';
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { fileURLToPath } from 'node:url';
4
4
  import sitemap, {} from '@astrojs/sitemap';
5
+ import { IMAGE_MAX_BYTES, largeImages, readBuiltPages, seoWarnings, } from "./built-pages.js";
5
6
  import { loadWerkbank } from "./content.js";
6
7
  import { designClasses, designCss, designFromSetup, mergeDesigns, validateDesign, } from "./design.js";
7
8
  import { occasionEntries } from "./occasions.js";
9
+ import { llmsTxt, robotsTxt, securityTxt } from "./root-files.js";
8
10
  import { prepareUploads, removeOriginals } from "./uploads.js";
9
11
  const CONFIG_MODULE = 'virtual:maverick-wave-astro/config';
10
12
  const WERKBANK_MODULE = 'virtual:maverick-wave-astro/werkbank';
@@ -48,14 +50,55 @@ export const buildConfig = (options, root, werkbank = loadWerkbank(root)) => {
48
50
  occasions: occasions ? occasionEntries(occasions) : [],
49
51
  };
50
52
  };
53
+ // Only a site at the root of its host: /.well-known and robots.txt below a
54
+ // base are never asked for.
55
+ const rootFiles = ({ config: { site }, site: url, base }, sitemap, pages, logger) => {
56
+ if (!url || base.replace(/\/$/, '') !== '')
57
+ return {};
58
+ const origin = new URL(url);
59
+ if (!site.email)
60
+ logger.info('no site.email, so no security.txt');
61
+ return {
62
+ ...(site.email && {
63
+ '.well-known/security.txt': securityTxt(origin, site.email, site.lang),
64
+ }),
65
+ ...(!site.noIndex && {
66
+ 'robots.txt': robotsTxt(origin, { aiCrawlers: site.aiCrawlers, sitemap }),
67
+ 'llms.txt': llmsTxt(site.name, site.lang, origin, pages),
68
+ }),
69
+ };
70
+ };
71
+ const writeRootFiles = (out, publicDir, files, logger) => {
72
+ for (const [file, content] of Object.entries(files)) {
73
+ if (existsSync(path.join(publicDir, file))) {
74
+ logger.warn(`public/${file} wins, delete it to get the generated one`);
75
+ continue;
76
+ }
77
+ const target = path.join(out, file);
78
+ mkdirSync(path.dirname(target), { recursive: true });
79
+ writeFileSync(target, content);
80
+ }
81
+ };
82
+ const reportSeo = (out, pages, logger) => {
83
+ for (const page of pages.filter(page => !page.noIndex)) {
84
+ for (const warning of seoWarnings(page)) {
85
+ logger.warn(`${new URL(page.url).pathname}: ${warning}`);
86
+ }
87
+ }
88
+ for (const { file, bytes } of largeImages(out)) {
89
+ logger.warn(`${file}: ${Math.round(bytes / 1024)} KB, keep images under ${IMAGE_MAX_BYTES / 1024} KB`);
90
+ }
91
+ };
51
92
  export default function homepage(options) {
93
+ let build;
52
94
  return {
53
95
  name: 'maverick-wave-astro',
54
96
  hooks: {
55
97
  'astro:config:setup': async ({ config: astroConfig, updateConfig, injectRoute, injectScript, addWatchFile, logger, }) => {
56
98
  const root = fileURLToPath(astroConfig.root);
57
99
  // Before any page renders: Picture reads the sizes from public/.
58
- const uploads = await prepareUploads(fileURLToPath(astroConfig.publicDir));
100
+ const publicDir = fileURLToPath(astroConfig.publicDir);
101
+ const uploads = await prepareUploads(publicDir);
59
102
  if (uploads.written > 0)
60
103
  logger.info(`${uploads.written} image sizes for uploads written`);
61
104
  for (const file of uploads.skipped)
@@ -65,6 +108,12 @@ export default function homepage(options) {
65
108
  logger.info(`start content for ${werkbank.fromStart.join(', ')} - nothing from the werkbank`);
66
109
  }
67
110
  const config = buildConfig(options, root, werkbank);
111
+ build = {
112
+ config,
113
+ site: astroConfig.site,
114
+ base: astroConfig.base,
115
+ publicDir,
116
+ };
68
117
  const modules = {
69
118
  [CONFIG_MODULE]: config,
70
119
  [WERKBANK_MODULE]: {
@@ -128,7 +177,16 @@ export default function homepage(options) {
128
177
  });
129
178
  }
130
179
  },
131
- 'astro:build:done': ({ dir }) => removeOriginals(fileURLToPath(dir)),
180
+ 'astro:build:done': ({ dir, logger }) => {
181
+ const out = fileURLToPath(dir);
182
+ removeOriginals(out);
183
+ const pages = readBuiltPages(out);
184
+ if (build) {
185
+ const files = rootFiles(build, options.sitemap !== false, pages, logger);
186
+ writeRootFiles(out, build.publicDir, files, logger);
187
+ }
188
+ reportSeo(out, pages, logger);
189
+ },
132
190
  },
133
191
  };
134
192
  }
@@ -0,0 +1,9 @@
1
+ import type { BuiltPage } from './built-pages.ts';
2
+ export type AiCrawlers = 'allow' | 'search-only';
3
+ export declare const robotsTxt: (site: URL, { aiCrawlers, sitemap, }: {
4
+ aiCrawlers?: AiCrawlers;
5
+ sitemap: boolean;
6
+ }) => string;
7
+ export declare const securityTxt: (site: URL, email: string, lang: string, built?: Date) => string;
8
+ /** The indexable pages as llmstxt.org lists them, start page first, then the FAQ. */
9
+ export declare const llmsTxt: (name: string, lang: string, site: URL, pages: readonly BuiltPage[]) => string;
@@ -0,0 +1,61 @@
1
+ // Collect training data. Google-Extended only covers Gemini - Search and the
2
+ // AI Overviews go through Googlebot, so blocking it costs no visibility.
3
+ const TRAINING_CRAWLERS = ['GPTBot', 'ClaudeBot', 'Google-Extended'];
4
+ // Search index and fetches on a user's request.
5
+ const SEARCH_CRAWLERS = [
6
+ 'OAI-SearchBot',
7
+ 'ChatGPT-User',
8
+ 'Claude-SearchBot',
9
+ 'Claude-User',
10
+ 'PerplexityBot',
11
+ ];
12
+ // RFC 9116 wants Expires less than a year ahead. The date freezes with the build.
13
+ const SECURITY_TXT_DAYS = 360;
14
+ const DAY = 24 * 60 * 60 * 1000;
15
+ const group = (agent, rule) => `User-agent: ${agent}\n${rule}`;
16
+ // A crawler without its own group follows "*" anyway. With 'allow' the groups
17
+ // change nothing, they write the decision down where checkers look for it.
18
+ export const robotsTxt = (site, { aiCrawlers = 'allow', sitemap, }) => {
19
+ const training = aiCrawlers === 'allow' ? 'Allow: /' : 'Disallow: /';
20
+ return `${[
21
+ group('*', 'Allow: /'),
22
+ ...TRAINING_CRAWLERS.map(agent => group(agent, training)),
23
+ ...SEARCH_CRAWLERS.map(agent => group(agent, 'Allow: /')),
24
+ ...(sitemap
25
+ ? [`Sitemap: ${new URL('/sitemap-index.xml', site).href}`]
26
+ : []),
27
+ ].join('\n\n')}\n`;
28
+ };
29
+ export const securityTxt = (site, email, lang, built = new Date()) => [
30
+ `Contact: mailto:${email}`,
31
+ `Expires: ${new Date(built.getTime() + SECURITY_TXT_DAYS * DAY).toISOString()}`,
32
+ `Preferred-Languages: ${lang}`,
33
+ `Canonical: ${new URL('/.well-known/security.txt', site).href}`,
34
+ '',
35
+ ].join('\n');
36
+ const link = (page) => `- [${page.title.replace(/[[\]]/g, '\\$&')}](${page.url})${page.description ? `: ${page.description}` : ''}`;
37
+ /** The indexable pages as llmstxt.org lists them, start page first, then the FAQ. */
38
+ export const llmsTxt = (name, lang, site, pages) => {
39
+ const german = lang.startsWith('de');
40
+ const home = new URL('/', site).href;
41
+ const rank = (page) => (page.url === home ? '' : page.url);
42
+ const listed = pages
43
+ .filter(page => !page.noIndex)
44
+ .sort((a, b) => rank(a).localeCompare(rank(b)));
45
+ const summary = listed.find(page => page.url === home)?.description;
46
+ const questions = [
47
+ ...new Map(listed
48
+ .flatMap(page => page.questions)
49
+ .map(entry => [entry.question, entry])).values(),
50
+ ];
51
+ return `${[
52
+ `# ${name}`,
53
+ summary && `> ${summary}`,
54
+ `## ${german ? 'Seiten' : 'Pages'}`,
55
+ listed.map(link).join('\n'),
56
+ questions.length > 0 && `## ${german ? 'Häufige Fragen' : 'Questions'}`,
57
+ ...questions.map(entry => `### ${entry.question}\n\n${entry.answer}`),
58
+ ]
59
+ .filter(Boolean)
60
+ .join('\n\n')}\n`;
61
+ };
package/dist/types.d.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import type { Design } from './design.ts';
2
2
  import type { OccasionEntry } from './occasions.ts';
3
+ import type { AiCrawlers } from './root-files.ts';
3
4
  export type Site = {
4
5
  name: string;
5
6
  lang?: string;
@@ -12,6 +13,8 @@ export type Site = {
12
13
  hasSvgFavicon?: boolean;
13
14
  colorScheme?: 'dark' | 'light';
14
15
  noIndex?: boolean;
16
+ email?: string;
17
+ aiCrawlers?: AiCrawlers;
15
18
  counter?: boolean;
16
19
  themeToggle?: 'system' | 'dark';
17
20
  mwScript?: boolean;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "maverick-wave-astro",
3
- "version": "1.8.0",
3
+ "version": "1.9.0",
4
4
  "description": "Shared frame for static Astro homepages built with MaverickWave: layout, SEO, design variants, schedule, building blocks.",
5
5
  "type": "module",
6
6
  "license": "MIT",