@webround/core 0.3.13 → 0.3.15

package/dist/context/WrAppContext.js

@@ -48,7 +48,7 @@ function WrRouteElement({ route, pageConfigMap, lazyFallback: LazyFallback, Page
             const Component = route.Component;
             return _jsx(Component, {});
         })();
-    return (_jsxs(PageProvider, { config: config, pageName: pageName, children: [_jsx(WrScroll, {}), _jsxs(WrSharedContext.Provider, { value: shared, children: [_jsx(WrSeoHead, { site: seo, config: config, pathname: pathname, lang: lang, langs: langs }), _jsx(WrLayout, { Header: Header, Footer: Footer, showHeader: route.header, showFooter: route.footer, pageConfigMap: pageConfigMap, ScopeProvider: PageProvider, children: body })] })] }, pageName));
+    return (_jsxs(PageProvider, { config: config, pageName: pageName, children: [_jsx(WrScroll, {}), _jsxs(WrSharedContext.Provider, { value: shared, children: [_jsx(WrSeoHead, { site: seo, config: config, home: pageConfigMap.home, pathname: pathname, lang: lang, langs: langs }), _jsx(WrLayout, { Header: Header, Footer: Footer, showHeader: route.header, showFooter: route.footer, pageConfigMap: pageConfigMap, ScopeProvider: PageProvider, children: body })] })] }, pageName));
 }
 /**
  * An address that leads to no page is the home page. The address is replaced BEFORE the router reads it, so the first render is the home page's: what was served for that address is the home page's

package/dist/kinds.d.ts

@@ -3,7 +3,7 @@ export declare const WR_KINDS: {
         readonly component: "WrSection";
         readonly tag: {
             readonly default: "section";
-            readonly allowed: readonly ["section", "div", "article", "aside", "header", "footer", "main", "nav"];
+            readonly allowed: readonly ["section", "div", "article", "aside", "header", "footer", "main", "nav", "ul", "ol", "li", "dl", "dt", "dd", "figure", "figcaption", "details", "summary", "address", "blockquote", "hgroup"];
         };
         readonly holds: "elements";
     };

package/dist/kinds.js

@@ -1,7 +1,7 @@
 // The one table that says what the Webround elements are. Everything that needs to know — the editor's engine, its field schema, the checks,
 // the components' own `tag` prop, the parent editor — derives from it: a new element, or a new tag, is written here and nowhere else.
 export const WR_KINDS = {
-    section: { component: 'WrSection', tag: { default: 'section', allowed: ['section', 'div', 'article', 'aside', 'header', 'footer', 'main', 'nav'] }, holds: 'elements' },
+    section: { component: 'WrSection', tag: { default: 'section', allowed: ['section', 'div', 'article', 'aside', 'header', 'footer', 'main', 'nav', 'ul', 'ol', 'li', 'dl', 'dt', 'dd', 'figure', 'figcaption', 'details', 'summary', 'address', 'blockquote', 'hgroup'] }, holds: 'elements' },
     text: { component: 'WrText', tag: { default: 'div', allowed: ['div', 'span'] }, holds: 'none' },
     image: { component: 'WrImage', tag: { fixed: 'img' }, holds: 'none' },
     link: { component: 'WrLink', tag: { fixed: 'a' }, holds: 'label' },

package/dist/seo-react.d.ts

@@ -10,9 +10,11 @@ export declare function useWrSeo(seo: WrSeoPage | undefined): void;
  * A page's tags for search engines and social networks, rendered where the page is: React puts `<title>`, `<meta>` and `<link>` in
  * `<head>` by itself. Both packages use this one, so what the editor shows is what production serves.
  */
-export declare function WrSeoHead({ site, config, pathname, lang, langs }: {
+export declare function WrSeoHead({ site, config, home, pathname, lang, langs }: {
     site?: WrSeoSite;
     config: WrPageConfig | undefined;
+    /** The home page's JSON: the structured data of a page that has none of its own. */
+    home?: WrPageConfig;
     pathname: string;
     lang: string;
     langs?: WrLangConfig;

package/dist/seo-react.js

@@ -39,10 +39,10 @@ export function useWrSeo(seo) {
  * A page's tags for search engines and social networks, rendered where the page is: React puts `<title>`, `<meta>` and `<link>` in
  * `<head>` by itself. Both packages use this one, so what the editor shows is what production serves.
  */
-export function WrSeoHead({ site, config, pathname, lang, langs }) {
+export function WrSeoHead({ site, config, home, pathname, lang, langs }) {
     const path = langs ? wrBuildLangPath('default', pathname, langs) : pathname;
     const dynamic = useSyncExternalStore(subscribe, () => dynamicSeo, () => undefined);
-    const { title, tags, jsonLd } = wrSeoTags({ site, page: { ...wrSeoPageOf(config), ...dynamic }, path, lang, langs });
+    const { title, tags, jsonLd } = wrSeoTags({ site, page: { ...wrSeoPageOf(config), ...dynamic }, home: wrSeoPageOf(home), path, lang, langs });
     // The `<title>` that `index.html` has (a placeholder, the page app's own) is not the page's: when the page has one of its own, the head must have only that, as the HTML a crawler gets says it
     useEffect(() => {
         if (title)

package/dist/seo.d.ts

@@ -26,6 +26,10 @@ export interface WrSeoSite {
     extraSitemaps?: string[];
     /** The code Google Search Console gives to prove the site is yours (the `content` of its `google-site-verification` tag): only the code, never the tag. It goes in `index.html` (vite-plugins/head.ts), not in the tags of `wrSeoTags`: a crawler looks for it before any script runs. */
     searchConsole?: string;
+    /** Whether the AI crawlers (the ones that train models and the ones that answer for a person) may read the site. Absent or `allow`: everybody may, as robots.txt always said. `block`: robots.txt says no to each of `WR_AI_BOTS`, and no `llms.txt` is written. robots.txt is a request, not a lock. */
+    aiBots?: 'allow' | 'block';
+    /** `false`: the site does not publish an `llms.txt` (a plain list of its pages, for AI assistants). Absent: it does. */
+    llms?: boolean;
 }
 /** What one page says (under `_seo` in its JSON). Text can be translated like every other text. */
 export interface WrSeoPage {
@@ -74,6 +78,11 @@ export type WrSeoJsonLd = WrJsonLd[] | Record<string, WrJsonLd[]>;
 export declare const WR_ROBOTS_FILE = "robots.txt";
 export declare const WR_SITEMAP_INDEX_FILE = "sitemap-index.xml";
 export declare const WR_SITEMAP_PAGES_FILE = "sitemap-pages.xml";
+/** The list of the site's pages for AI assistants (llmstxt.org). Ends with `WR_LLMS_MARK`: a file without it is somebody's own, and is never overwritten nor removed. */
+export declare const WR_LLMS_FILE = "llms.txt";
+export declare const WR_LLMS_MARK = "<!-- Written by Webround from the pages of the site: change them, not this file. -->";
+/** The user agents of the AI crawlers `aiBots: 'block'` turns away: the ones that gather text to train models and the ones that fetch pages for an assistant. */
+export declare const WR_AI_BOTS: readonly ["GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "anthropic-ai", "PerplexityBot", "Perplexity-User", "Google-Extended", "Applebot-Extended", "Amazonbot", "Meta-ExternalAgent", "CCBot", "Bytespider", "cohere-ai", "Diffbot"];
 /** The sitemaps the shop's server writes every day (not the build): a site with the page of a product (or of a collection) always has its index point to the sitemap of them. */
 export declare const WR_SITEMAP_PRODUCTS_FILE = "sitemap-products.xml";
 export declare const WR_SITEMAP_COLLECTIONS_FILE = "sitemap-collections.xml";
@@ -105,11 +114,18 @@ export interface WrSeoContext {
     /** The language shown: `default`, or a code. */
     lang: string;
     langs?: WrLangConfig;
+    /** The home page's own SEO data: the structured data of a page that has none of its own is the home's. */
+    home?: WrSeoPage;
 }
 /** The text of the tab: the page's title in the template, or the site's name when the page has none. */
 export declare function wrSeoTitle({ site, page, lang }: Pick<WrSeoContext, 'site' | 'page' | 'lang'>): string | undefined;
 /** Everything a page puts in `<head>` for search engines and social networks: ONE function, whoever renders or injects it. */
 export declare function wrSeoTags(context: WrSeoContext): WrSeoHead;
+/**
+ * What a head says twice: a `<meta>` (by its name or property) or a `<link>` (by its `rel`, and language) that more than one tag writes, and a structured-data type that is meant to be
+ * said once. A page's `meta` list can repeat what the page already writes (a second description), and a site's and the home's structured data can both be an organisation.
+ */
+export declare function wrSeoDuplicates(head: Pick<WrSeoHead, 'tags' | 'jsonLd'>): string[];
 /** A page of the site as a sitemap sees it: its address in every language. */
 export interface WrSeoEntry {
     pageName: string;
@@ -158,10 +174,21 @@ export declare function wrSnapshotTargets(input: {
 export declare function wrSitemapPagesXml(entries: readonly WrSeoEntry[], langs?: WrLangConfig): string;
 /** The index of the sitemaps: the site's own, and whichever others the site says exist (e.g. the shop's, written by the server). */
 export declare function wrSitemapIndexXml(siteUrl: string, files: readonly string[]): string;
-export declare function wrRobotsTxt(siteUrl: string): string;
+export declare function wrRobotsTxt(siteUrl: string, aiBots?: WrSeoSite['aiBots']): string;
+/**
+ * The `llms.txt` of a site: its name, what its home says about it and, for each language, the pages a search engine may index (title, address and description). Plain
+ * Markdown, the shape llmstxt.org asks for. Pages are listed as the sitemap lists them: the same entries, the same exclusions.
+ */
+export declare function wrLlmsTxt(input: {
+    entries: readonly WrSeoEntry[];
+    pages: Record<string, WrPageConfig | undefined>;
+    langs?: WrLangConfig;
+    site?: WrSeoSite;
+    siteUrl: string;
+}): string;
 /**
- * The files a site has for search engines, by name (relative to `public/`): robots.txt, the index and the sitemap of the site's pages.
- * `null` while the site does not know its public address: there is nothing true to write.
+ * The files a site has for search engines and AI assistants, by name (relative to `public/`): robots.txt, the index and the sitemap of the site's pages, and `llms.txt` (unless the site
+ * turned it off or blocks the AI crawlers). `null` while the site does not know its public address: there is nothing true to write.
  */
 export declare function wrSeoFiles(input: {
     routes: WrRoutesFile;

package/dist/seo.js

@@ -20,6 +20,14 @@ export function wrSearchConsoleCode(input) {
 export const WR_ROBOTS_FILE = 'robots.txt';
 export const WR_SITEMAP_INDEX_FILE = 'sitemap-index.xml';
 export const WR_SITEMAP_PAGES_FILE = 'sitemap-pages.xml';
+/** The list of the site's pages for AI assistants (llmstxt.org). Ends with `WR_LLMS_MARK`: a file without it is somebody's own, and is never overwritten nor removed. */
+export const WR_LLMS_FILE = 'llms.txt';
+export const WR_LLMS_MARK = '<!-- Written by Webround from the pages of the site: change them, not this file. -->';
+/** The user agents of the AI crawlers `aiBots: 'block'` turns away: the ones that gather text to train models and the ones that fetch pages for an assistant. */
+export const WR_AI_BOTS = [
+    'GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'Claude-SearchBot', 'Claude-User', 'anthropic-ai', 'PerplexityBot', 'Perplexity-User',
+    'Google-Extended', 'Applebot-Extended', 'Amazonbot', 'Meta-ExternalAgent', 'CCBot', 'Bytespider', 'cohere-ai', 'Diffbot',
+];
 /** The sitemaps the shop's server writes every day (not the build): a site with the page of a product (or of a collection) always has its index point to the sitemap of them. */
 export const WR_SITEMAP_PRODUCTS_FILE = 'sitemap-products.xml';
 export const WR_SITEMAP_COLLECTIONS_FILE = 'sitemap-collections.xml';
@@ -135,9 +143,32 @@ export function wrSeoTags(context) {
         else if (extra.name?.trim())
             meta('name', extra.name.trim(), content);
     }
-    const jsonLd = [...wrSeoJsonLdOf(site?.jsonLd, lang), ...wrSeoJsonLdOf(page?.jsonLd, lang)].map(wrJsonLdText);
+    // The page's own structured data; a page that has none uses the home's, but only what is about the whole site (who the organisation is): a FAQ, a product or an article of the home says nothing about
+    // another page, and structured data that is not about what the page shows is against the search engines' rules. A site whose home has none has none (nothing is made up).
+    const own = wrSeoJsonLdOf(page?.jsonLd, lang);
+    // The same object written at the site and at the home is one object, not two
+    const jsonLd = [...new Set([...wrSeoJsonLdOf(site?.jsonLd, lang), ...(own.length > 0 ? own : wrSeoJsonLdOf(context.home?.jsonLd, lang).filter(isSiteWideJsonLd))].map(wrJsonLdText))];
     return { title, tags, jsonLd };
 }
+/** The structured-data types that are about the whole site, and that a page is meant to say once: two of them are two answers to "who is this?". */
+const WR_SITE_WIDE_JSON_LD_TYPES = ['Organization', 'WebSite', 'LocalBusiness'];
+const typesOfJsonLd = (item) => [item['@type']].flat().filter((one) => typeof one === 'string');
+const isSiteWideJsonLd = (item) => typesOfJsonLd(item).some(type => WR_SITE_WIDE_JSON_LD_TYPES.includes(type));
+/**
+ * What a head says twice: a `<meta>` (by its name or property) or a `<link>` (by its `rel`, and language) that more than one tag writes, and a structured-data type that is meant to be
+ * said once. A page's `meta` list can repeat what the page already writes (a second description), and a site's and the home's structured data can both be an organisation.
+ */
+export function wrSeoDuplicates(head) {
+    const count = (keys) => keys.reduce((seen, key) => seen.set(key, (seen.get(key) ?? 0) + 1), new Map());
+    const tags = count(head.tags.map(({ tag, attrs }) => tag === 'meta'
+        ? `<meta ${attrs.name ? `name="${attrs.name}"` : `property="${attrs.property}"`}>`
+        : `<link rel="${attrs.rel}"${attrs.hreflang ? ` hreflang="${attrs.hreflang}"` : ''}>`));
+    const types = count(head.jsonLd.flatMap(text => typesOfJsonLd(JSON.parse(text)).filter(type => WR_SITE_WIDE_JSON_LD_TYPES.includes(type))));
+    return [
+        ...[...tags].filter(([, n]) => n > 1).map(([key, n]) => `${key} is written ${n} times`),
+        ...[...types].filter(([, n]) => n > 1).map(([type, n]) => `the structured data has ${n} objects of type ${type}`),
+    ];
+}
 /**
  * The pages a sitemap can list: those with a fixed address (a `[slug]` route is a family of pages the shop's own sitemap tells about).
  * Pages marked noindex are kept in the list (the checks and the editor show them) and left out by `wrSitemapPagesXml`.
@@ -204,12 +235,36 @@ export function wrSitemapIndexXml(siteUrl, files) {
     const rows = files.map(file => `  <sitemap>\n    <loc>${xml(isAbsolute(file) ? file : `${siteUrl}/${file.replace(/^\/+/, '')}`)}</loc>\n  </sitemap>`);
     return `${XML_HEAD}<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${rows.join('\n')}\n</sitemapindex>\n`;
 }
-export function wrRobotsTxt(siteUrl) {
-    return `User-agent: *\nAllow: /\n\nSitemap: ${siteUrl}/${WR_SITEMAP_INDEX_FILE}\n`;
+export function wrRobotsTxt(siteUrl, aiBots = 'allow') {
+    const blocked = aiBots === 'block' ? `${WR_AI_BOTS.map(bot => `User-agent: ${bot}`).join('\n')}\nDisallow: /\n\n` : '';
+    return `${blocked}User-agent: *\nAllow: /\n\nSitemap: ${siteUrl}/${WR_SITEMAP_INDEX_FILE}\n`;
+}
+/**
+ * The `llms.txt` of a site: its name, what its home says about it and, for each language, the pages a search engine may index (title, address and description). Plain
+ * Markdown, the shape llmstxt.org asks for. Pages are listed as the sitemap lists them: the same entries, the same exclusions.
+ */
+export function wrLlmsTxt(input) {
+    const { entries, pages, langs, site, siteUrl } = input;
+    const keyOf = (lang) => (lang === langs?.default ? 'default' : lang);
+    const name = wrLocalized(site?.siteName, 'default')?.trim() || new URL(siteUrl).host;
+    const about = wrLocalized(wrSeoPageOf(pages.home).description, 'default')?.trim();
+    const indexed = entries.filter(entry => !entry.noindex);
+    const sections = (langs && langs.available.length > 0 ? langs.available : ['default']).map((code) => {
+        const key = keyOf(code);
+        const items = indexed.map((entry) => {
+            const seo = wrSeoPageOf(pages[entry.pageName]);
+            const url = entry.urls.find(u => u.lang === code)?.url ?? entry.urls[0].url;
+            const title = wrLocalized(seo.title, key)?.trim() || (entry.pageName === 'home' ? wrLocalized(site?.siteName, key)?.trim() || name : entry.pageName);
+            const description = wrLocalized(seo.description, key)?.trim();
+            return `- [${title}](${url})${description ? `: ${description}` : ''}`;
+        });
+        return `## ${langs && langs.available.length > 1 ? `Pages (${code})` : 'Pages'}\n\n${items.join('\n')}`;
+    });
+    return `# ${name}\n\n${about ? `> ${about}\n\n` : ''}${sections.join('\n\n')}\n\n${WR_LLMS_MARK}\n`;
 }
 /**
- * The files a site has for search engines, by name (relative to `public/`): robots.txt, the index and the sitemap of the site's pages.
- * `null` while the site does not know its public address: there is nothing true to write.
+ * The files a site has for search engines and AI assistants, by name (relative to `public/`): robots.txt, the index and the sitemap of the site's pages, and `llms.txt` (unless the site
+ * turned it off or blocks the AI crawlers). `null` while the site does not know its public address: there is nothing true to write.
  */
 export function wrSeoFiles(input) {
     const siteUrl = wrSiteUrlOf(input.site);
@@ -219,10 +274,12 @@ export function wrSeoFiles(input) {
     // The shop's sitemaps are cited whether the files exist yet or not: the server writes them daily and the index must already say where they are
     const own = [WR_SITEMAP_PAGES_FILE, ...Object.entries(WR_SHOP_SITEMAPS).filter(([route]) => route in input.routes).map(([, file]) => file)];
     const extra = (input.site?.extraSitemaps ?? []).filter(name => typeof name === 'string' && name.trim() && !own.includes(name));
+    const llms = input.site?.llms !== false && input.site?.aiBots !== 'block';
     return {
-        [WR_ROBOTS_FILE]: wrRobotsTxt(siteUrl),
+        [WR_ROBOTS_FILE]: wrRobotsTxt(siteUrl, input.site?.aiBots),
         [WR_SITEMAP_INDEX_FILE]: wrSitemapIndexXml(siteUrl, [...own, ...extra]),
         [WR_SITEMAP_PAGES_FILE]: wrSitemapPagesXml(entries, input.langs),
+        ...(llms ? { [WR_LLMS_FILE]: wrLlmsTxt({ entries, pages: input.pages, langs: input.langs, site: input.site, siteUrl }) } : {}),
     };
 }
 //# sourceMappingURL=seo.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@webround/core",
-    "version": "0.3.13",
+    "version": "0.3.15",
     "type": "module",
     "main": "./dist/index.js",
     "module": "./dist/index.js",