@webround/editor 0.3.14 → 0.3.15

package/dist/check/producer-code.js

@@ -3,8 +3,8 @@ import path from 'path';
 import { Project, SyntaxKind } from 'ts-morph';
 import { WR_COMPONENT_TO_KIND, wrTagOf } from '@webround/core/kinds';
 import { wrAriaLabel, wrLocalized } from '@webround/core/localized';
-import { WR_SHARED_SCOPES, wrPageConfigPath } from '@webround/core/routing';
-import { wrJsonLdProblems, wrSeoPageOf } from '@webround/core/seo';
+import { WR_SHARED_SCOPES, wrPageConfigPath, wrRoutePattern } from '@webround/core/routing';
+import { wrJsonLdProblems, wrSeoDuplicates, wrSeoPageOf, wrSeoTags } from '@webround/core/seo';
 import { findPageRoot, syntaxErrors } from '../vite-plugins/jsx-tree.js';
 import { createOperationContext } from '../operations/context.js';
 import { readJsonFile, readLangs, readRoutes, readSeoSite, scopesOfPage, sharedScopesOf } from '../operations/ops/project.js';
@@ -309,10 +309,18 @@ export function checkScopeFromCode(ctx, scope, options = {}) {
     return { name: scope.name, file: scope.file, nodes, syntax };
 }
 /** What a page tells search engines, as the rules read it: every text by language key (a plain string is the default's). */
-function seoOfPage(ctx, name) {
+function seoOfPage(ctx, name, lang) {
     const seo = wrSeoPageOf(readJsonFile(ctx, wrPageConfigPath(name)) ?? undefined);
     const byLang = (value) => typeof value === 'string' ? { default: value } : { ...(value ?? {}) };
-    return { title: byLang(seo.title), description: byLang(seo.description), noindex: seo.noindex === true, family: /\[[^\]]+\]/.test(name), jsonLdProblems: wrJsonLdProblems(seo.jsonLd) };
+    // The head the page really gets (the same function the page renders), to see what it says twice
+    const head = wrSeoTags({
+        site: readSeoSite(ctx), page: seo, home: wrSeoPageOf(readJsonFile(ctx, wrPageConfigPath('home')) ?? undefined),
+        path: wrRoutePattern(name), lang, langs: readLangs(ctx) ?? undefined,
+    });
+    return {
+        title: byLang(seo.title), description: byLang(seo.description), noindex: seo.noindex === true, family: /\[[^\]]+\]/.test(name),
+        jsonLdProblems: wrJsonLdProblems(seo.jsonLd), duplicates: wrSeoDuplicates(head),
+    };
 }
 /** What a link may lead to: the routes, the languages and the files served as they are. */
 function projectOf(ctx) {
@@ -345,7 +353,7 @@ export function checkPagesFromCode(root, lang = 'default') {
         const syntax = [...scopes.flatMap(scope => scope.syntax), ...follow.syntax];
         return {
             name, nodes: scopes.flatMap(scope => scope.nodes), ...(syntax.length ? { syntax } : {}),
-            ...(multilingual ? { lang } : {}), project, labelledIds, seo: seoOfPage(ctx, name),
+            ...(multilingual ? { lang } : {}), project, labelledIds, seo: seoOfPage(ctx, name, lang),
         };
     });
 }
@@ -368,7 +376,17 @@ function readIndexHtml(root) {
     const lang = /<html\b[^>]*\blang\s*=\s*["']([^"']+)["']/i.exec(html)?.[1] ?? null;
     const viewport = /<meta\b[^>]*\bname\s*=\s*["']viewport["'][^>]*\bcontent\s*=\s*["']([^"']*)["']/i.exec(html)?.[1]
         ?? /<meta\b[^>]*\bcontent\s*=\s*["']([^"']*)["'][^>]*\bname\s*=\s*["']viewport["']/i.exec(html)?.[1] ?? null;
-    return { lang, viewport };
+    // What each page writes by itself: if `index.html` has it, it is in every page and the page's own comes with it
+    const seoTags = [...html.matchAll(/<(meta|link)\b[^>]*>/gi)].flatMap(([tag, kind]) => {
+        const attr = (name) => new RegExp(`\\b${name}\\s*=\\s*["']([^"']*)["']`, 'i').exec(tag)?.[1]?.toLowerCase();
+        const key = kind.toLowerCase() === 'link' ? (attr('rel') === 'canonical' ? 'canonical' : undefined) : attr('name') ?? attr('property');
+        if (key === 'canonical')
+            return ['<link rel="canonical">'];
+        if (key === 'description')
+            return ['<meta name="description">'];
+        return key && /^(og|twitter):/.test(key) ? [`<meta ${attr('name') ? 'name' : 'property'}="${key}">`] : [];
+    });
+    return { lang, viewport, seoTags: [...new Set(seoTags)] };
 }
 /** What the checks know of the site as a whole. */
 export function checkSiteFromCode(root) {

package/dist/check/rules.js

@@ -143,6 +143,14 @@ export const RULES = [
                 report(null, `The structured data of the page is not valid: ${problem}. Search engines ignore it.`);
         },
     },
+    {
+        name: 'seo-duplicates',
+        severity: 'warning',
+        run(page, report) {
+            for (const problem of page.seo?.duplicates ?? [])
+                report(null, `The head of the page repeats itself: ${problem}. Search engines and social networks may read the wrong one: keep one.`, undefined, problem.replace(/\d+ (times|objects)/, '$1'));
+        },
+    },
     {
         name: 'broken-link',
         severity: 'warning',
@@ -461,6 +469,14 @@ export const SITE_RULES = [
         run: (site, report) => { for (const problem of wrJsonLdProblems(site.seo.jsonLd))
             report(`The structured data of the site is not valid: ${problem}. Search engines ignore it.`, undefined); },
     },
+    {
+        name: 'index-seo-tags',
+        severity: 'warning',
+        run: (site, report) => {
+            for (const tag of site.index?.seoTags ?? [])
+                report(`index.html has ${tag}: it is in every page, the same, and each page writes its own as well, so the head has two. Take it out of index.html and write it in the SEO panel.`, undefined, undefined, tag);
+        },
+    },
     {
         name: 'focus-visible-css',
         severity: 'warning',

package/dist/check/types.d.ts

@@ -66,6 +66,8 @@ export interface CheckPage {
         family: boolean;
         /** What is wrong with the page's JSON-LD (empty when it is fine). */
         jsonLdProblems: string[];
+        /** What the page's head says twice (`wrSeoDuplicates`), in the language the page was read in. */
+        duplicates: string[];
     };
 }
 /** Where a link of the site may lead. */
@@ -110,7 +112,8 @@ export interface CheckSite {
     /** What `index.html` says (the page every route is served in); null while the project has none. */
     index: {
         lang: string | null;
-        viewport: string | null;
+        viewport: string | null; /** The tags the pages write themselves (description, canonical, Open Graph, Twitter) that `index.html` already has: they would be in every page, twice. */
+        seoTags: string[];
     } | null;
     /** Some element has an id the code computes: a key that looks unused may belong to it, so nothing can be said to be unused. */
     dynamicIds: boolean;

package/dist/context/WrAppContext.js

@@ -56,7 +56,7 @@ function WrRouteElement({ route, lazyFallback: LazyFallback, Header, Footer, })
             const Component = route.Component;
             return _jsx(Component, {});
         })();
-    return (_jsxs(WrPageProvider, { config: config, pageName: pageName, file: file, children: [_jsx(WrScroll, {}), _jsx(WrSeoHead, { site: site, config: config, pathname: pathname, lang: lang, langs: langs }), _jsx(WrLayout, { Header: Header, Footer: Footer, showHeader: route.header, showFooter: route.footer, children: body })] }, pageName));
+    return (_jsxs(WrPageProvider, { config: config, pageName: pageName, file: file, children: [_jsx(WrScroll, {}), _jsx(WrSeoHead, { site: site, config: config, home: pageConfigMap.home, pathname: pathname, lang: lang, langs: langs }), _jsx(WrLayout, { Header: Header, Footer: Footer, showHeader: route.header, showFooter: route.footer, children: body })] }, pageName));
 }
 export function WrAppProvider({ routesFile, eagerModules, lazyModules, configModules, langs, initialLang = 'default', lazyFallback = WrDefaultPageFallback, Header, Footer, seo, shared, children, }) {
     // Config delle pagine: stato live, indipendente dal router.

package/dist/messages.d.ts

@@ -162,6 +162,10 @@ export interface WrSeoSiteSetRequest {
     image?: string;
     /** Il codice (o il tag intero) di verifica di Google Search Console. */
     searchConsole?: string;
+    /** `block`: robots.txt manda via i crawler AI (e non si scrive `llms.txt`); `allow` è com'era. */
+    aiBots?: 'allow' | 'block';
+    /** `false`: il sito non pubblica `llms.txt`. */
+    llms?: boolean;
     twitterSite?: string;
     twitterCreator?: string;
     /** Per `lang` (default se omessa): nome, modello del titolo, immagine e dati strutturati. */

package/dist/operations/ops/seo.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import { wrPageConfigPath } from '@webround/core/routing';
-import { WR_SEO_KEY, wrJsonLdProblems, wrSearchConsoleCode, wrSeoFiles, wrSnapshotTargets, wrSeoPageOf, wrSiteUrlOf } from '@webround/core/seo';
+import { WR_LLMS_FILE, WR_LLMS_MARK, WR_SEO_KEY, wrJsonLdProblems, wrSearchConsoleCode, wrSeoFiles, wrSnapshotTargets, wrSeoPageOf, wrSiteUrlOf } from '@webround/core/seo';
 import { wrLocalizeItems, wrSetLocalized } from '@webround/core/localized';
 import { defineOperation, flag, invalid, list, OperationError, text } from '../define.js';
 import { checkScopeFromCode } from '../../check/producer-code.js';
@@ -40,13 +40,23 @@ export function writeSeoFiles(ctx) {
         return { written: [], reason: 'no-site-url' };
     const written = [];
     fs.mkdirSync(ctx.publicRoot, { recursive: true });
+    // An `llms.txt` without our mark is somebody's own: it is neither replaced nor removed
+    const llmsFile = path.join(ctx.publicRoot, WR_LLMS_FILE);
+    const ownLlms = fs.existsSync(llmsFile) && !fs.readFileSync(llmsFile, 'utf-8').includes(WR_LLMS_MARK);
     for (const [name, content] of Object.entries(files)) {
         const file = path.join(ctx.publicRoot, name);
+        if (name === WR_LLMS_FILE && ownLlms)
+            continue;
         if (fs.existsSync(file) && fs.readFileSync(file, 'utf-8') === content)
             continue;
         fs.writeFileSync(file, content);
         written.push(name);
     }
+    // The site no longer has one (turned off, or the AI crawlers are blocked): the one we wrote goes
+    if (!(WR_LLMS_FILE in files) && fs.existsSync(llmsFile) && !ownLlms) {
+        fs.rmSync(llmsFile);
+        written.push(`${WR_LLMS_FILE} (removed)`);
+    }
     return { written };
 }
 export const seoGet = defineOperation({
@@ -277,6 +287,8 @@ export const seoSiteSet = defineOperation({
         { name: 'image', type: 'string', help: 'the sharing picture of pages that have none: a path in the site or a full address (written in `lang`)' },
         { name: 'searchConsole', type: 'string', help: 'the code that proves the site is yours to Google Search Console: the code, or the whole <meta name="google-site-verification"…> tag as Google shows it (only the code is kept); it is written in every page\'s head' },
         { name: 'extraSitemaps', type: 'array', help: 'sitemaps the index must point to besides the site\'s own, e.g. ["sitemap-products.xml"] (the shop writes it); a list, [] for none' },
+        { name: 'aiBots', type: 'string', choices: ['allow', 'block'], help: 'whether AI crawlers (GPTBot, ClaudeBot, PerplexityBot, Google-Extended…) may read the site: allow (the default) or block, which robots.txt asks them to respect and which also turns the llms.txt off' },
+        { name: 'llms', type: 'boolean', help: 'publish an llms.txt, a plain list of the site\'s pages for AI assistants (on unless turned off with false)' },
         { name: 'twitterSite', type: 'string', help: 'the account of the site on X/Twitter (@acme), for every page' },
         { name: 'twitterCreator', type: 'string', help: 'the account of the author on X/Twitter (@jane), for every page' },
         { name: 'lang', type: 'string', help: 'the language of `siteName`, `titleTemplate`, `image` and `jsonLd`: `default` (the default one) or a language code of the site' },
@@ -317,6 +329,15 @@ export const seoSiteSet = defineOperation({
                 throw invalid('a sitemap is a file name or an address, without spaces', 'invalid-sitemap');
             patch.extraSitemaps = names.length > 0 ? names : null;
         }
+        const aiBots = text(input, 'aiBots');
+        if (aiBots !== undefined) {
+            if (aiBots.trim() && !['allow', 'block'].includes(aiBots.trim()))
+                throw invalid('aiBots is allow or block', 'invalid-value');
+            patch.aiBots = aiBots.trim() === 'block' ? 'block' : null;
+        }
+        const llms = flag(input, 'llms');
+        if (llms !== undefined)
+            patch.llms = llms ? null : false;
         if (input.twitterSite !== undefined || input.twitterCreator !== undefined) {
             const twitter = { ...(readSeoSite(ctx).twitter ?? {}) };
             for (const [param, key] of [['twitterSite', 'site'], ['twitterCreator', 'creator']]) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@webround/editor",
-    "version": "0.3.14",
+    "version": "0.3.15",
     "type": "module",
     "main": "./dist/index.js",
     "module": "./dist/index.js",
@@ -75,7 +75,7 @@
         "@lexical/selection": "^0.51.0",
         "@lexical/utils": "^0.51.0",
         "@monaco-editor/react": "^4.7.0",
-        "@webround/core": "^0.3.14",
+        "@webround/core": "^0.3.15",
         "highlight.js": "^11.12.0",
         "hls.js": "^1.5.0",
         "lexical": "^0.51.0",