@nutin/cli 2.1.1 → 3.0.0

package/templates/base/tools/builder/core/prod-bundle/bundle-options.test.js.hbs

@@ -0,0 +1,65 @@
+import { sharedDefine, slimNutinConfig, slimSeoConfig, isPlainData } from './bundle-options.js';
+
+describe('bundle-options', () => {
+  it('sharedDefine sets NODE_ENV, the i18n flag and the markdown sources from the build config', () => {
+    expect(sharedDefine({ isProd: true, i18n: false })).toEqual({
+      'process.env.NODE_ENV': '"production"',
+      'globalThis.__NUTIN_I18N__': 'false',
+      'globalThis.__NUTIN_MARKDOWN_SOURCES__': JSON.stringify('[]'),
+    });
+    expect(sharedDefine({ isProd: false, i18n: true, markdownSources: { sourceFolders: ['docs', { folder: 'x', seo: false }] } })).toEqual({
+      'process.env.NODE_ENV': '"development"',
+      'globalThis.__NUTIN_I18N__': 'true',
+      'globalThis.__NUTIN_MARKDOWN_SOURCES__': JSON.stringify('["docs",{"folder":"x","seo":false}]'),
+    });
+  });
+
+  it('slimNutinConfig drops tooling-only keys and keeps everything app code may read', () => {
+    const slim = slimNutinConfig({
+      i18n: true,
+      generateSEOFiles: false,
+      markdownSources: { sourceFolders: ['docs'] },
+      myAppKey: 'kept',
+      builder: { esbuild: { minify: true } },
+      testinNutin: { includeApp: false },
+      dockerPorts: [9090],
+    });
+
+    expect(slim).toEqual({
+      i18n: true,
+      generateSEOFiles: false,
+      myAppKey: 'kept',
+    });
+  });
+
+  it('slimNutinConfig returns null when the remaining config is not plain data', () => {
+    expect(slimNutinConfig({ i18n: false, onBuild: () => {} })).toBe(null);
+    expect(slimNutinConfig({ i18n: false, pattern: /x/ })).toBe(null);
+  });
+
+  it('slimNutinConfig ignores non-plain values inside tooling-only keys', () => {
+    expect(slimNutinConfig({ i18n: false, builder: { plugin: () => {} } })).toEqual({ i18n: false });
+  });
+
+  it('slimSeoConfig keeps only each route\'s path and title when SEO generation is enabled', () => {
+    const seo = {
+      baseUrl: 'https://example.com',
+      disallowBots: ['GPTBot'],
+      routes: [{ path: '/', title: { en: 'Home' }, description: 'd', ogImage: '/og.jpg' }],
+    };
+
+    expect(slimSeoConfig(seo, true)).toEqual({ routes: [{ path: '/', title: { en: 'Home' } }] });
+  });
+
+  it('slimSeoConfig returns an empty object when SEO generation is disabled or routes are missing', () => {
+    expect(slimSeoConfig({ routes: [{ path: '/', title: 'Home' }] }, false)).toEqual({});
+    expect(slimSeoConfig({}, true)).toEqual({ routes: [] });
+  });
+
+  it('isPlainData accepts JSON-like values only', () => {
+    expect(isPlainData({ a: [1, 'b', true, null, { c: 2 }] })).toBeTruthy();
+    expect(isPlainData(undefined)).toBeFalsy();
+    expect(isPlainData(new Date())).toBeFalsy();
+    expect(isPlainData([() => {}])).toBeFalsy();
+  });
+});

package/templates/base/tools/builder/core/prod-bundle/compress-files.js.hbs

@@ -1,13 +1,13 @@
 import { readFileSync, writeFileSync } from 'fs';
 import { gzipSync, brotliCompressSync, constants } from 'zlib';
-import { print, getFilesRecursive, errorExit } from '../../../utils/index.js';
-import { PATHS } from '../app/paths.js';
+import { getFilesRecursive, errorExit } from '../../../utils/index.js';
+import { PATHS, isBundledSource } from '../app/paths.js';
 
 function compressStaticAssets() {
   const files = getFilesRecursive(
     PATHS.tempSource,
-    ['.js', '.css', '.json', '.svg', '.ttf', '.otf', '.eot']
-  );
+    ['.html', '.js', '.css', '.json', '.svg', '.xml', '.txt', '.ttf', '.otf', '.eot']
+  ).filter((file) => !isBundledSource(file));
   
   for (const file of files) {
     const content = readFileSync(file);
@@ -27,11 +27,11 @@ function compressStaticAssets() {
       }
     });
     writeFileSync(`${file}.br`, brotlied);
-  };
+  }
 }
 
 try {
-  compressStaticAssets()
+  compressStaticAssets();
 } catch(err) {
   errorExit(err, 'compress-files');
 }

package/templates/base/tools/builder/core/prod-bundle/esbuild.js.hbs

@@ -1,13 +1,18 @@
 import esbuild from 'esbuild';
 import path from 'path';
-import { print, errorExit } from '../../../utils/index.js';
+import { errorExit } from '../../../utils/index.js';
 import { PATHS } from '../app/paths.js';
 import { builderConfig } from '../../builder.config.js';
+import { sharedDefine, slimConfigPlugin } from './bundle-options.js';
 
+{{#if (eq lang "js")}}
+// Bundles the copied .js sources; in dev they're also kept on disk for testin-nutin.
+const ENTRY_FILE = path.join(PATHS.tempApp, 'main.js');
+{{else}}
 // Prod bundles straight from the copied .ts sources (tsc runs --noEmit there);
 // dev bundles tsc's emitted .js, which is kept on disk for testin-nutin.
 const ENTRY_FILE = path.join(PATHS.tempApp, builderConfig.isProd ? 'main.ts' : 'main.js');
-const OUT_FILE = path.join(PATHS.tempSource, 'bundle.js');
+{{/if}}
 
 const DEV_OVERRIDES = {
   minify: false,
@@ -25,11 +30,14 @@ async function build() {
     loader: {
       '.json': 'json',
     },
-    define: {
-      'process.env.NODE_ENV': builderConfig.isProd ? '"production"' : '"development"',
-    },
-    entryPoints: [ENTRY_FILE],
-    outfile: OUT_FILE,
+    define: sharedDefine(builderConfig),
+    plugins: [slimConfigPlugin({ generateSEO: builderConfig.generateSEO })],
+    // Code splitting: each dynamic import() becomes a chunk in chunks/, only fetched
+    // the first time it runs. The entry still comes out as bundle.js (see add-tags.js).
+    splitting: true,
+    entryPoints: [{ in: ENTRY_FILE, out: 'bundle' }],
+    outdir: PATHS.tempSource,
+    chunkNames: `${path.basename(PATHS.tempChunks)}/[name]-[hash]`,
     keepNames: false,
   });
 }

package/templates/base/tools/builder/core/prod-bundle/hash-files.js.hbs

@@ -1,11 +1,11 @@
 import fs from 'fs';
 import path from 'path';
 import crypto from 'crypto';
-import { PATHS } from '../app/paths.js';
-import { print, getFilesRecursive, errorExit } from '../../../utils/index.js';
+import { PATHS, isBundledSource, isChunk } from '../app/paths.js';
+import { getFilesRecursive, errorExit } from '../../../utils/index.js';
 
 function addHashToFiles(distDir) {
-  const files = getFilesRecursive(distDir, ['.js', '.css']);
+  const files = getFilesRecursive(distDir, ['.js', '.css']).filter((file) => !isBundledSource(file) && !isChunk(file));
 
   const fileMap = {}; // Track old → new filenames
 
@@ -49,7 +49,7 @@ function updateHtmlReferences(htmlPath, fileMap) {
 }
 
 try {
-  addHashToFiles(PATHS.tempSource)
+  addHashToFiles(PATHS.tempSource);
 } catch(err) {
   errorExit(err, 'hash-files');
 }

package/templates/base/tools/builder/core/seo/generate-robots-txt.js.hbs

@@ -1,29 +1,36 @@
 import fs from 'fs';
 import path from 'path';
 import { PATHS } from '../app/paths.js';
-import { print } from '../../../utils/index.js';
+import { builderConfig } from '../../builder.config.js';
 
-export function generateRobotsTxt() {
-  const seoConfigPath = path.join(PATHS.temp, 'config', 'seo.json');
-  const seoConfig = JSON.parse(fs.readFileSync(seoConfigPath, 'utf-8'));
+// The URL path(s) a route's Disallow lines must cover: `:param` segments become `*` (every
+// value of that segment), and with i18n on the pages live under each /<lang>/ prefix.
+function disallowPathsFor(route, { i18n, languages }) {
+  const pattern = route.path.replace(/:[^/]+/g, '*');
+  if (!i18n) return [pattern];
+  const suffix = pattern === '/' ? '/' : pattern;
+  return languages.map((lang) => `/${lang}${suffix}`);
+}
+
+export function buildRobotsTxt(seoConfig, { i18n = false, languages = [] } = {}) {
+  const i18nOptions = { i18n, languages };
   const baseUrl = seoConfig.baseUrl.replace(/\/$/, '');
   const routes = seoConfig.routes ?? [];
-  const disallowBots = seoConfig.disallowBots ?? [];
+  const disallowBots = new Set(seoConfig.disallowBots ?? []);
 
   // Paths disallowed for all bots (disallow: true)
   const globalDisallows = routes
     .filter(r => r.disallow === true)
-    .map(r => `Disallow: ${r.path}`);
+    .flatMap(r => disallowPathsFor(r, i18nOptions));
 
   // Paths disallowed per named bot: { botName -> [path, ...] }
   const perBotDisallows = {};
   for (const r of routes) {
     if (!Array.isArray(r.disallow)) continue;
     for (const bot of r.disallow) {
-      (perBotDisallows[bot] ??= []).push(r.path);
+      (perBotDisallows[bot] ??= []).push(...disallowPathsFor(r, i18nOptions));
     }
   }
-  // Bots that are fully blocked contribute an empty-path disallow
   for (const bot of disallowBots) {
     perBotDisallows[bot] ??= [];
   }
@@ -31,21 +38,32 @@ export function generateRobotsTxt() {
   const lines = [
     'User-agent: *',
     'Allow: /',
-    ...globalDisallows,
+    ...globalDisallows.map(p => `Disallow: ${p}`),
   ];
 
   for (const [bot, paths] of Object.entries(perBotDisallows)) {
     lines.push('');
     lines.push(`User-agent: ${bot}`);
-    if (paths.length > 0) {
-      for (const p of paths) lines.push(`Disallow: ${p}`);
-    } else {
+    if (disallowBots.has(bot)) {
+      // Fully blocked — per-route entries for this bot are redundant.
       lines.push('Disallow: /');
+      continue;
     }
+    // A bot with its own group ignores the `*` group entirely, so it has to
+    // repeat the global disallows to stay blocked from them.
+    for (const p of new Set([...globalDisallows, ...paths])) lines.push(`Disallow: ${p}`);
   }
 
   lines.push('', `Sitemap: ${baseUrl}/sitemap.xml`);
+  return lines.join('\n') + '\n';
+}
+
+export function generateRobotsTxt() {
+  const seoConfigPath = path.join(PATHS.temp, 'config', 'seo.json');
+  const seoConfig = JSON.parse(fs.readFileSync(seoConfigPath, 'utf-8'));
+  const languagesConfigPath = path.join(PATHS.temp, 'config', 'languages.json');
+  const { languages } = JSON.parse(fs.readFileSync(languagesConfigPath, 'utf-8'));
 
   const outputPath = path.join(PATHS.tempSource, 'robots.txt');
-  fs.writeFileSync(outputPath, lines.join('\n') + '\n', 'utf-8');
+  fs.writeFileSync(outputPath, buildRobotsTxt(seoConfig, { i18n: builderConfig.i18n, languages }), 'utf-8');
 }

package/templates/base/tools/builder/core/seo/generate-robots-txt.test.js.hbs

@@ -0,0 +1,48 @@
+import { buildRobotsTxt } from './generate-robots-txt.js';
+
+const groupOf = (txt, bot) => {
+  const lines = txt.split('\n');
+  const start = lines.indexOf(`User-agent: ${bot}`);
+  const end = lines.indexOf('', start);
+  return lines.slice(start + 1, end === -1 ? undefined : end);
+};
+
+describe('buildRobotsTxt', () => {
+  it('a bot in disallowBots is fully blocked even when a route also names it', () => {
+    const txt = buildRobotsTxt({
+      baseUrl: 'https://site.test/',
+      disallowBots: ['GPTBot'],
+      routes: [{ path: '/private', disallow: ['GPTBot'] }],
+    });
+
+    expect(groupOf(txt, 'GPTBot')).toEqual(['Disallow: /']);
+  });
+
+  it('a bot with its own group repeats the global disallows', () => {
+    const txt = buildRobotsTxt({
+      baseUrl: 'https://site.test',
+      routes: [
+        { path: '/admin', disallow: true },
+        { path: '/drafts', disallow: ['Bingbot'] },
+      ],
+    });
+
+    expect(groupOf(txt, '*')).toEqual(['Allow: /', 'Disallow: /admin']);
+    expect(groupOf(txt, 'Bingbot')).toEqual(['Disallow: /admin', 'Disallow: /drafts']);
+  });
+
+  it('turns :param segments into wildcards and prefixes every language with i18n on', () => {
+    const txt = buildRobotsTxt(
+      { baseUrl: 'https://site.test', routes: [{ path: '/users/:id', disallow: true }] },
+      { i18n: true, languages: ['en', 'fr'] }
+    );
+
+    expect(groupOf(txt, '*')).toEqual(['Allow: /', 'Disallow: /en/users/*', 'Disallow: /fr/users/*']);
+  });
+
+  it('ends with the sitemap URL', () => {
+    const txt = buildRobotsTxt({ baseUrl: 'https://site.test/', routes: [] });
+
+    expect(txt.endsWith('Sitemap: https://site.test/sitemap.xml\n')).toBeTruthy();
+  });
+});

package/templates/base/tools/builder/core/seo/generate-seo-files.js.hbs

@@ -1,12 +1,12 @@
-import { print, errorExit } from '../../../utils/index.js';
+import { errorExit } from '../../../utils/index.js';
 import { generateSeoHtml } from './generate-seo-html.js';
 import { generateRobotsTxt } from './generate-robots-txt.js';
 import { generateSitemapXml } from './generate-sitemap-xml.js';
 
 async function generateSeoFiles() {
-  await generateSeoHtml();
+  const skippedSuffixes = await generateSeoHtml();
   generateRobotsTxt();
-  generateSitemapXml();
+  generateSitemapXml(skippedSuffixes);
 }
 
 try {

package/templates/base/tools/builder/core/seo/generate-seo-html.js.hbs

@@ -6,7 +6,8 @@ import { buildSsrBundle, cleanupSsrBundle } from './ssr/ssr-bundle.js';
 import { renderRoute } from './ssr/ssr-render.js';
 import { builderConfig } from '../../builder.config.js';
 import { resolveLocaleValue, valueForLangWithFallback, collectI18nSeoIssues } from './i18n-resolution.js';
-import { segmentsOf, validateMockParams, writeRouteHtml, warnForRoutesMissingSeoConfig } from './route-output.js';
+import { segmentsOf, routeSuffixOf, validateMockParams, writeRouteHtml, warnForRoutesMissingSeoConfig } from './route-output.js';
+import { resolveSeoRoutes, resolveSeoOptOutPaths } from './seo-routes.js';
 
 // og:image/twitter:image require an absolute URL for social platforms to resolve them;
 // every other absolute field (og:url, canonical) is already baseUrl-prefixed, so ogImage
@@ -16,10 +17,11 @@ function toAbsoluteUrl(urlOrPath, baseUrl) {
   return /^https?:\/\//.test(urlOrPath) ? urlOrPath : `${baseUrl}${urlOrPath}`;
 }
 
+// Returns the guard result when a route guard refused the route (nothing written), else null.
 async function processRoute(template, baseUrl, defaultLanguage, languages, route, bundleUrl) {
   validateMockParams(route);
 
-  const routeSuffix = route.path === '/' ? '' : route.path;
+  const routeSuffix = routeSuffixOf(route);
 
   if (builderConfig.i18n) {
     // Same for every language variant of this route — lets crawlers know these URLs
@@ -30,13 +32,12 @@ async function processRoute(template, baseUrl, defaultLanguage, languages, route
       { hreflang: 'x-default', href: `${baseUrl}/${defaultLanguage}${routeSuffix}/` },
     ];
 
+    // Every language is rendered before anything is written: a guard refusing any of
+    // them skips the whole route, rather than shipping some language variants.
+    const pages = [];
     for (const lang of languages) {
-      const title = valueForLangWithFallback(route.title, lang, defaultLanguage);
-      const description = valueForLangWithFallback(route.description, lang, defaultLanguage);
-      const ogImage = toAbsoluteUrl(valueForLangWithFallback(route.ogImage, lang, defaultLanguage), baseUrl);
-
       const pageUrl = `${baseUrl}/${lang}${routeSuffix}`;
-      const body = await renderRoute({
+      const result = await renderRoute({
         bundleUrl,
         appRoutesKey: route.path,
         mockParams: route.mockParams,
@@ -45,6 +46,14 @@ async function processRoute(template, baseUrl, defaultLanguage, languages, route
         pageUrl,
         i18nEnabled: builderConfig.i18n,
       });
+      if (result.blocked) return result;
+      pages.push({ lang, pageUrl, body: result.html });
+    }
+
+    for (const { lang, pageUrl, body } of pages) {
+      const title = valueForLangWithFallback(route.title, lang, defaultLanguage);
+      const description = valueForLangWithFallback(route.description, lang, defaultLanguage);
+      const ogImage = toAbsoluteUrl(valueForLangWithFallback(route.ogImage, lang, defaultLanguage), baseUrl);
 
       await writeRouteHtml({
         template, lang, title, description, pageUrl, ogImage, body, hreflangLinks,
@@ -62,7 +71,7 @@ async function processRoute(template, baseUrl, defaultLanguage, languages, route
     }
 
     const pageUrl = `${baseUrl}${routeSuffix}`;
-    const body = await renderRoute({
+    const result = await renderRoute({
       bundleUrl,
       appRoutesKey: route.path,
       mockParams: route.mockParams,
@@ -71,13 +80,24 @@ async function processRoute(template, baseUrl, defaultLanguage, languages, route
       pageUrl,
       i18nEnabled: builderConfig.i18n,
     });
+    if (result.blocked) return result;
 
     await writeRouteHtml({
-      template, lang: defaultLanguage, title, description, pageUrl, ogImage, body,
+      template, lang: defaultLanguage, title, description, pageUrl, ogImage, body: result.html,
       outputSegments: segmentsOf(routeSuffix),
       routePath: route.path,
     });
   }
+
+  return null;
+}
+
+function describeBlocked(route, { redirectTo }) {
+  const outcome = redirectTo ? `redirected an anonymous visitor to "${redirectTo}"` : 'blocked an anonymous visitor';
+  const url = routeSuffixOf(route) || '/';
+  const where = url === route.path ? `"${route.path}"` : `"${route.path}" (${url})`;
+  return `[generate-seo-html] Route ${where} was not prerendered: its guard ${outcome}. ` +
+    `It is left out of sitemap.xml too.`;
 }
 
 export async function generateSeoHtml() {
@@ -131,14 +151,24 @@ export async function generateSeoHtml() {
     }
   }
 
+  // routeSuffixOf() of every route a guard refused, so the sitemap can leave them out too.
+  const skippedSuffixes = new Set();
+
   try {
     const bundleUrl = await buildSsrBundle();
-    await warnForRoutesMissingSeoConfig(bundleUrl, seoConfig.routes, defaultLanguage, baseUrl);
-
-    for (const route of seoConfig.routes) {
-      await processRoute(template, baseUrl, defaultLanguage, languages, route, bundleUrl);
+    // seo.json routes plus feature-generated ones (e.g. one per Markdown page).
+    const routes = resolveSeoRoutes(seoConfig);
+    await warnForRoutesMissingSeoConfig(bundleUrl, routes, defaultLanguage, baseUrl, resolveSeoOptOutPaths());
+
+    for (const route of routes) {
+      const blocked = await processRoute(template, baseUrl, defaultLanguage, languages, route, bundleUrl);
+      if (!blocked) continue;
+      print.warn(describeBlocked(route, blocked));
+      skippedSuffixes.add(routeSuffixOf(route));
     }
   } finally {
     cleanupSsrBundle();
   }
+
+  return skippedSuffixes;
 }

package/templates/base/tools/builder/core/seo/generate-sitemap-xml.js.hbs

@@ -1,16 +1,22 @@
 import fs from 'fs';
 import path from 'path';
 import { PATHS } from '../app/paths.js';
-import { print } from '../../../utils/index.js';
 import { builderConfig } from '../../builder.config.js';
+import { routeSuffixOf } from './route-output.js';
+import { resolveSeoRoutes } from './seo-routes.js';
 
-function collectUrls(baseUrl, routes, languages) {
+// `skippedSuffixes`: routeSuffixOf() of routes generate-seo-html.js didn't prerender
+// because a guard refused them.
+export function collectUrls(baseUrl, routes, languages, { i18n = builderConfig.i18n, skippedSuffixes = new Set() } = {}) {
   const urls = [];
 
   for (const route of routes) {
-    const routeSuffix = route.path === '/' ? '' : route.path;
+    // Disallowed for every bot in robots.txt — listing it here would contradict that.
+    if (route.disallow === true) continue;
+    const routeSuffix = routeSuffixOf(route);
+    if (skippedSuffixes.has(routeSuffix)) continue;
 
-    if (builderConfig.i18n) {
+    if (i18n) {
       for (const lang of languages) {
         urls.push(`${baseUrl}/${lang}${routeSuffix}/`);
       }
@@ -22,12 +28,21 @@ function collectUrls(baseUrl, routes, languages) {
   return urls;
 }
 
+function escapeXml(str) {
+  return str
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;');
+}
+
 function buildSitemapXml(urls) {
-  const urlEntries = urls.map(loc => `  <url>\n    <loc>${loc}</loc>\n  </url>`).join('\n');
+  const urlEntries = urls.map(loc => `  <url>\n    <loc>${escapeXml(loc)}</loc>\n  </url>`).join('\n');
   return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urlEntries}\n</urlset>\n`;
 }
 
-export function generateSitemapXml() {
+export function generateSitemapXml(skippedSuffixes = new Set()) {
   const seoConfigPath = path.join(PATHS.temp, 'config', 'seo.json');
   const seoConfig = JSON.parse(fs.readFileSync(seoConfigPath, 'utf-8'));
   const baseUrl = seoConfig.baseUrl.replace(/\/$/, '');
@@ -35,7 +50,7 @@ export function generateSitemapXml() {
   const languagesConfigPath = path.join(PATHS.temp, 'config', 'languages.json');
   const { languages } = JSON.parse(fs.readFileSync(languagesConfigPath, 'utf-8'));
 
-  const urls = collectUrls(baseUrl, seoConfig.routes, languages);
+  const urls = collectUrls(baseUrl, resolveSeoRoutes(seoConfig), languages, { skippedSuffixes });
   const xml = buildSitemapXml(urls);
 
   const outputPath = path.join(PATHS.tempSource, 'sitemap.xml');

package/templates/base/tools/builder/core/seo/generate-sitemap-xml.test.js.hbs

@@ -0,0 +1,30 @@
+import { collectUrls } from './generate-sitemap-xml.js';
+
+const ROUTES = [
+  { path: '/' },
+  { path: '/admin' },
+  { path: '/users/:id', mockParams: { id: '42' } },
+  { path: '/drafts', disallow: true },
+];
+
+describe('collectUrls', () => {
+  it('lists every route at its prerendered URL, minus routes disallowed for every bot', () => {
+    expect(collectUrls('https://site.test', ROUTES, ['en'], { i18n: false })).toEqual([
+      'https://site.test/',
+      'https://site.test/admin/',
+      'https://site.test/users/42/',
+    ]);
+  });
+
+  it('leaves out routes a guard kept from being prerendered', () => {
+    const urls = collectUrls('https://site.test', ROUTES, ['en'], { i18n: false, skippedSuffixes: new Set(['/admin']) });
+
+    expect(urls).toEqual(['https://site.test/', 'https://site.test/users/42/']);
+  });
+
+  it('skips a guarded route in every language with i18n on', () => {
+    const urls = collectUrls('https://site.test', ROUTES.slice(0, 2), ['en', 'fr'], { i18n: true, skippedSuffixes: new Set(['/admin']) });
+
+    expect(urls).toEqual(['https://site.test/en/', 'https://site.test/fr/']);
+  });
+});