@hagicode/hagilight 0.9.5-dev.109.1.aafdacc → 0.9.7-dev.111.1.899dd08

package/README.md

@@ -49,6 +49,30 @@ The values come from `HAGILIGHT_BUILD_BRANCH` and `HAGILIGHT_BUILD_COMMIT`, then
 
 `github.head_ref` is set on pull requests and `github.ref_name` on pushes. A shallow clone still reports its commit, because `git rev-parse HEAD` reads the checkout.
 
+## llms.txt
+
+`hagilight()` writes `/llms.txt` after the build, following the [llms.txt](https://llmstxt.org/) layout: the site name as the H1, the home page description as a blockquote summary, and one H2 section of links per top-level path segment. It is generated from the built HTML, so it works for plain Astro and Starlight sites alike and needs no content collection.
+
+- Only one language is listed: the Astro `i18n.defaultLocale`, then the home page language. The home pages of the other languages go under `## Optional`.
+- Pages are left out when they are `noindex`, instant redirects, 404 pages, or declare a canonical URL for another page.
+- Each link carries the page's meta description. A description repeated on three or more pages is treated as a site-wide default and left out of the links.
+- Section order is Overview first, then by page count. A section lists 12 links (Overview 36) and the file lists at most 120.
+- An `llms.txt` that the site already provides, as a public file or a page, is never overwritten. `hagilight({ enabled: false })` skips generation too.
+
+```js
+hagilight({
+  llms: {
+    title: 'Example Docs',        // default: the home page og:site_name
+    summary: 'What this site is.', // default: the home page description
+    lang: 'en-US',                // default: i18n.defaultLocale
+    maxLinksPerSection: 60,       // default: 12
+    maxLinks: 200,                // default: 120
+  },
+});
+```
+
+Pass `llms: false` (or `{ enabled: false }`) to opt out. The file is only as good as the page titles and descriptions it reads, so write a real `description` for important pages.
+
 ## Body images
 
 `hagilight()` registers the content-image plugin from `@hagicode/hagilight-core/content-images`. Markdown and MDX body images then get `loading="lazy"`, `decoding="async"`, and the class `hagilight-content-image`, with the author's values kept. This registration does not depend on `enabled`, so `hagilight({ enabled: false })` still registers the plugin.

package/dist/integration.d.ts

@@ -21,6 +21,26 @@ export interface HagilightOptions {
      * Independent of `enabled`, which controls only sitemap and robots.
      */
     contentImages?: boolean | HagilightContentImagesOptions;
+    /**
+     * Generate `/llms.txt` after the build from the built pages. Defaults to `true`; `false` or
+     * `{ enabled: false }` opts out. A site-provided `llms.txt` (public file or page) is never
+     * overwritten. It also follows `enabled`: `hagilight({ enabled: false })` skips it.
+     */
+    llms?: boolean | HagilightLlmsOptions;
+}
+export interface HagilightLlmsOptions {
+    /** Defaults to `true`. */
+    enabled?: boolean;
+    /** Site name for the H1. Defaults to the home page `og:site_name` or title. */
+    title?: string;
+    /** Summary blockquote. Defaults to the home page description. */
+    summary?: string;
+    /** Language to list, for example `en-US`. Defaults to the Astro default locale, then the home page language. */
+    lang?: string;
+    /** Maximum links per section. Defaults to 12. */
+    maxLinksPerSection?: number;
+    /** Maximum links in total. Defaults to 120. */
+    maxLinks?: number;
 }
 export interface HagilightContentImagesOptions {
     /** Defaults to `true`. */
@@ -39,6 +59,6 @@ export type HagilightFaviconOptions = FaviconOptions;
  * owns sitemap generation, a consumer-owned robots page or public file takes
  * precedence, and an enabled Starlight Hagilight integration owns RSS feeds.
  */
-export declare function hagilight({ enabled, rss, contentImages }?: HagilightOptions): AstroIntegration;
+export declare function hagilight({ enabled, rss, contentImages, llms }?: HagilightOptions): AstroIntegration;
 /** Inject the bundled HagiCode favicon unless the site head already declares an icon. */
 export declare function hagilightFavicon(options?: HagilightFaviconOptions): AstroIntegration;

package/dist/integration.js

@@ -1,12 +1,13 @@
-import { existsSync, readdirSync, statSync } from 'node:fs';
+import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
-import { extname, isAbsolute, resolve, win32 } from 'node:path';
+import { extname, isAbsolute, join, resolve, win32 } from 'node:path';
 import sitemap from '@astrojs/sitemap';
 import { resolveBuildInfo } from '@hagicode/hagilight-core/build-info-resolver';
 import { checkContentImageRegistration, registerContentImages } from '@hagicode/hagilight-core/content-images';
 import { resolveFaviconHeadEntry } from '@hagicode/hagilight-core/favicon';
 import { resolveRssLocales } from '@hagicode/hagilight-core/rss';
 import { resolvePlainAstroRssOwner } from '@hagicode/hagilight-core/rss-ownership';
+import { buildLlmsTxt, parsePageMeta } from './llms.js';
 const VIRTUAL_RSS_CONFIG = 'virtual:hagilight/rss-config';
 const VIRTUAL_RSS_CONFIG_RESOLVED = `\0${VIRTUAL_RSS_CONFIG}`;
 const VIRTUAL_BUILD_INFO = 'virtual:hagilight/build-info';
@@ -51,7 +52,7 @@ const BUILTIN_RSS_FEED_SOURCE = BUILTIN_RSS_FEED.toString();
  * owns sitemap generation, a consumer-owned robots page or public file takes
  * precedence, and an enabled Starlight Hagilight integration owns RSS feeds.
  */
-export function hagilight({ enabled = true, rss, contentImages } = {}) {
+export function hagilight({ enabled = true, rss, contentImages, llms } = {}) {
     if (typeof enabled !== 'boolean') {
         throw new TypeError('Hagilight sitemap and robots enabled option must be a boolean.');
     }
@@ -61,6 +62,7 @@ export function hagilight({ enabled = true, rss, contentImages } = {}) {
     if (isRecord(contentImages) && contentImages.enabled !== undefined && typeof contentImages.enabled !== 'boolean') {
         throw new TypeError('Hagilight content images enabled option must be a boolean.');
     }
+    const llmsOptions = resolveLlmsOptions(llms);
     const contentImagesEnabled = typeof contentImages === 'boolean' ? contentImages : contentImages?.enabled !== false;
     if (rss !== undefined && typeof rss !== 'boolean' && !isRecord(rss)) {
         throw new TypeError('Hagilight RSS option must be a boolean or an options object.');
@@ -73,6 +75,7 @@ export function hagilight({ enabled = true, rss, contentImages } = {}) {
             throw new TypeError('Hagilight RSS getFeed must be a non-empty module path.');
         }
     }
+    let resolvedConfig;
     return {
         name: '@hagicode/hagilight:discovery',
         hooks: {
@@ -130,11 +133,79 @@ export function hagilight({ enabled = true, rss, contentImages } = {}) {
                 });
             },
             'astro:config:done'({ config, logger }) {
+                resolvedConfig = config;
                 checkContentImageRegistration({ config, logger, enabled: contentImagesEnabled });
             },
+            'astro:build:done'({ dir, pages, logger }) {
+                if (!enabled || !llmsOptions.enabled || !resolvedConfig)
+                    return;
+                writeLlmsTxt({ config: resolvedConfig, dir, pages, options: llmsOptions, logger });
+            },
         },
     };
 }
+function resolveLlmsOptions(llms) {
+    if (llms !== undefined && typeof llms !== 'boolean' && !isRecord(llms)) {
+        throw new TypeError('Hagilight llms option must be a boolean or an options object.');
+    }
+    if (typeof llms === 'boolean')
+        return { enabled: llms };
+    const options = llms ?? {};
+    if (options.enabled !== undefined && typeof options.enabled !== 'boolean') {
+        throw new TypeError('Hagilight llms enabled option must be a boolean.');
+    }
+    for (const key of ['title', 'summary', 'lang']) {
+        if (options[key] !== undefined && (typeof options[key] !== 'string' || !options[key].trim())) {
+            throw new TypeError(`Hagilight llms ${key} must be a non-empty string.`);
+        }
+    }
+    for (const key of ['maxLinksPerSection', 'maxLinks']) {
+        if (options[key] !== undefined && (!Number.isInteger(options[key]) || options[key] < 1)) {
+            throw new TypeError(`Hagilight llms ${key} must be a positive integer.`);
+        }
+    }
+    return { ...options, enabled: options.enabled !== false };
+}
+function writeLlmsTxt({ config, dir, pages, options, logger }) {
+    const outDir = fileURLToPath(dir);
+    const target = join(outDir, 'llms.txt');
+    if (existsSync(target)) {
+        logger.info('llms.txt already exists in the build output; leaving it untouched.');
+        return;
+    }
+    const base = `${config.base || '/'}`.replace(/\/?$/u, '/');
+    const siteBase = new URL(base, config.site);
+    const i18n = config.i18n;
+    const entries = [];
+    for (const { pathname } of pages) {
+        const relative = pathname.replace(/^\/+/u, '');
+        const candidates = relative === '' || relative.endsWith('/')
+            ? [join(outDir, relative, 'index.html')]
+            : [join(outDir, relative, 'index.html'), join(outDir, `${relative}.html`)];
+        const file = candidates.find((candidate) => existsSync(candidate) && statSync(candidate).isFile());
+        if (!file)
+            continue;
+        const url = new URL(relative, siteBase).href;
+        const meta = parsePageMeta(readFileSync(file, 'utf8'), url);
+        if (!meta)
+            continue;
+        entries.push({ path: new URL(url).pathname.slice(siteBase.pathname.length - 1), url, lang: meta.lang, title: meta.title, description: meta.description, siteName: meta.siteName });
+    }
+    const body = buildLlmsTxt({
+        pages: entries,
+        title: options.title,
+        summary: options.summary,
+        lang: options.lang ?? i18n?.defaultLocale,
+        maxLinksPerSection: options.maxLinksPerSection,
+        maxLinks: options.maxLinks,
+    });
+    if (!body) {
+        logger.warn('No indexable pages were found, so llms.txt was not generated.');
+        return;
+    }
+    writeFileSync(target, body, 'utf8');
+    logger.info(`Generated llms.txt from ${entries.length} page(s).`);
+}
 function deriveRssLocales(locales, config) {
     if (locales !== undefined)
         return locales;

package/dist/llms.d.ts

@@ -0,0 +1,43 @@
+/**
+ * `llms.txt` generation. The file is derived from the built HTML so it works for every site
+ * type (plain Astro, Starlight) without depending on content collections: each indexable page
+ * contributes its `<title>` and meta description, grouped by its first path segment.
+ * Format: https://llmstxt.org/ (H1 name, blockquote summary, H2 link sections).
+ */
+export interface LlmsPage {
+    /** Path relative to the site root, with a leading slash, for example `/guides/install/`. */
+    path: string;
+    url: string;
+    lang: string | undefined;
+    title: string;
+    description: string | undefined;
+    /** The page's `og:site_name`, used to name the site and to strip the title suffix. */
+    siteName?: string | undefined;
+}
+export interface LlmsPageMeta {
+    lang: string | undefined;
+    title: string;
+    description: string | undefined;
+    siteName: string | undefined;
+}
+export interface BuildLlmsTxtOptions {
+    pages: readonly LlmsPage[];
+    /** Overrides the site name taken from the home page. */
+    title?: string;
+    /** Overrides the summary taken from the home page description. */
+    summary?: string;
+    /** Preferred language, for example the Astro default locale. Falls back to the home page language. */
+    lang?: string;
+    maxLinksPerSection?: number;
+    maxLinks?: number;
+}
+export declare const DEFAULT_MAX_LINKS_PER_SECTION = 12;
+export declare const DEFAULT_MAX_LINKS = 120;
+/**
+ * Read the facts `llms.txt` needs from one built page. Returns `undefined` for pages that should
+ * not be listed: `noindex`, instant redirects, 404 pages, and pages whose canonical URL points
+ * somewhere else.
+ */
+export declare function parsePageMeta(html: string, pageUrl: string): LlmsPageMeta | undefined;
+/** Build the `llms.txt` body. Returns `undefined` when there is nothing to list. */
+export declare function buildLlmsTxt(options: BuildLlmsTxtOptions): string | undefined;

package/dist/llms.js

@@ -0,0 +1,183 @@
+/**
+ * `llms.txt` generation. The file is derived from the built HTML so it works for every site
+ * type (plain Astro, Starlight) without depending on content collections: each indexable page
+ * contributes its `<title>` and meta description, grouped by its first path segment.
+ * Format: https://llmstxt.org/ (H1 name, blockquote summary, H2 link sections).
+ */
+export const DEFAULT_MAX_LINKS_PER_SECTION = 12;
+export const DEFAULT_MAX_LINKS = 120;
+const OVERVIEW_LIMIT_FACTOR = 3;
+const GENERIC_DESCRIPTION_MIN_PAGES = 3;
+const LOCALE_SEGMENT = /^[a-z]{2,3}(?:-[a-z0-9]{2,8})?$/iu;
+const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
+function decodeEntities(value) {
+    return value.replace(/&(?:#(\d+)|#x([0-9a-f]+)|([a-z]+));/giu, (match, dec, hex, name) => {
+        if (dec)
+            return String.fromCodePoint(Number(dec));
+        if (hex)
+            return String.fromCodePoint(parseInt(hex, 16));
+        return ENTITIES[String(name).toLowerCase()] ?? match;
+    });
+}
+function attributes(tag) {
+    const result = {};
+    for (const [, name, double, single] of tag.matchAll(/([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/gu)) {
+        result[name.toLowerCase()] = decodeEntities(double ?? single ?? '');
+    }
+    return result;
+}
+function clean(value) {
+    const text = value?.replace(/\s+/gu, ' ').trim();
+    return text ? text : undefined;
+}
+function normalizeUrl(href, base) {
+    try {
+        const url = new URL(href, base);
+        url.hash = '';
+        url.search = '';
+        return url.href.replace(/\/index\.html$/u, '/');
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Read the facts `llms.txt` needs from one built page. Returns `undefined` for pages that should
+ * not be listed: `noindex`, instant redirects, 404 pages, and pages whose canonical URL points
+ * somewhere else.
+ */
+export function parsePageMeta(html, pageUrl) {
+    const headEnd = html.indexOf('</head>');
+    const head = headEnd === -1 ? html : html.slice(0, headEnd);
+    const metas = [...head.matchAll(/<meta\b[^>]*>/giu)].map(([tag]) => attributes(tag));
+    const named = (name) => metas.find((meta) => meta.name?.toLowerCase() === name)?.content;
+    const property = (name) => metas.find((meta) => meta.property?.toLowerCase() === name)?.content;
+    if (/noindex/iu.test(named('robots') ?? ''))
+        return undefined;
+    if (metas.some((meta) => meta['http-equiv']?.toLowerCase() === 'refresh'))
+        return undefined;
+    const canonicalTag = [...head.matchAll(/<link\b[^>]*>/giu)]
+        .map(([tag]) => attributes(tag))
+        .find((link) => link.rel?.toLowerCase().split(/\s+/u).includes('canonical'));
+    if (canonicalTag?.href) {
+        const canonical = normalizeUrl(canonicalTag.href, pageUrl);
+        const own = normalizeUrl(pageUrl, pageUrl);
+        // A trailing-slash difference is a site setting, not a different page.
+        if (canonical && own && canonical.replace(/\/$/u, '') !== own.replace(/\/$/u, ''))
+            return undefined;
+    }
+    const title = clean(decodeEntities(head.match(/<title[^>]*>([\s\S]*?)<\/title>/iu)?.[1] ?? ''));
+    if (!title || /^(?:404\b|page not found)/iu.test(title))
+        return undefined;
+    const htmlTag = html.match(/<html\b[^>]*>/iu)?.[0];
+    const lang = clean(htmlTag ? attributes(htmlTag).lang : undefined);
+    return {
+        lang,
+        title,
+        description: clean(named('description') ?? property('og:description')),
+        siteName: clean(property('og:site_name')),
+    };
+}
+function sameLanguage(a, b) {
+    if (!a || !b)
+        return false;
+    const x = a.toLowerCase();
+    const y = b.toLowerCase();
+    return x === y || x.startsWith(`${y}-`) || y.startsWith(`${x}-`);
+}
+function segmentsOf(page) {
+    const segments = page.path.split('/').filter(Boolean);
+    if (segments.length > 0 && LOCALE_SEGMENT.test(segments[0]) && (sameLanguage(segments[0], page.lang) || segments.length === 1)) {
+        segments.shift();
+    }
+    return segments;
+}
+function humanize(segment) {
+    const text = decodeURIComponent(segment).replace(/[-_]+/gu, ' ').trim();
+    return text.replace(/\b[a-z]/gu, (letter) => letter.toUpperCase());
+}
+function stripSiteSuffix(title, siteName) {
+    if (!siteName)
+        return title;
+    const escaped = siteName.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+    const stripped = title.replace(new RegExp(`\\s*[|•·–—-]\\s*${escaped}$`, 'iu'), '').trim();
+    return stripped || title;
+}
+function mostCommonLanguage(pages) {
+    const counts = new Map();
+    for (const page of pages)
+        if (page.lang)
+            counts.set(page.lang, (counts.get(page.lang) ?? 0) + 1);
+    return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]?.[0];
+}
+function mostCommonSiteName(pages) {
+    const counts = new Map();
+    for (const page of pages)
+        if (page.siteName)
+            counts.set(page.siteName, (counts.get(page.siteName) ?? 0) + 1);
+    return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]?.[0];
+}
+function linkLine(page, siteName, generic) {
+    const title = stripSiteSuffix(page.title, siteName).replace(/[[\]]/gu, '');
+    return page.description && !generic.has(page.description) ? `- [${title}](${page.url}): ${page.description}` : `- [${title}](${page.url})`;
+}
+/** Build the `llms.txt` body. Returns `undefined` when there is nothing to list. */
+export function buildLlmsTxt(options) {
+    const maxPerSection = options.maxLinksPerSection ?? DEFAULT_MAX_LINKS_PER_SECTION;
+    const maxLinks = options.maxLinks ?? DEFAULT_MAX_LINKS;
+    const all = [...options.pages].sort((a, b) => a.path.localeCompare(b.path));
+    if (all.length === 0)
+        return undefined;
+    const shallowest = (candidates) => [...candidates].sort((a, b) => segmentsOf(a).length - segmentsOf(b).length || a.path.length - b.path.length)[0];
+    let lang = options.lang && all.some((page) => sameLanguage(page.lang, options.lang))
+        ? options.lang
+        : shallowest(all.filter((page) => segmentsOf(page).length === 0))?.lang ?? mostCommonLanguage(all);
+    if (!all.some((page) => sameLanguage(page.lang, lang)))
+        lang = mostCommonLanguage(all);
+    const inLanguage = all.filter((page) => lang === undefined || sameLanguage(page.lang, lang));
+    const home = shallowest(inLanguage.filter((page) => segmentsOf(page).length === 0)) ?? shallowest(inLanguage);
+    const siteName = clean(options.title) ?? home?.siteName ?? mostCommonSiteName(inLanguage) ?? home?.title;
+    const summary = clean(options.summary) ?? home?.description;
+    const sections = new Map();
+    const hasChildren = (segment) => inLanguage.some((other) => {
+        const parts = segmentsOf(other);
+        return parts[0] === segment && parts.length > 1;
+    });
+    for (const page of inLanguage) {
+        const segments = segmentsOf(page);
+        // Top-level pages without children (for example /about/) go to Overview; /guides/... goes to Guides.
+        const key = segments.length === 0 || (segments.length === 1 && !hasChildren(segments[0]))
+            ? 'Overview'
+            : humanize(segments[0]);
+        sections.set(key, [...(sections.get(key) ?? []), page]);
+    }
+    // A description repeated across pages is a site-wide default, not information about the page.
+    const descriptionCounts = new Map();
+    for (const page of inLanguage)
+        if (page.description)
+            descriptionCounts.set(page.description, (descriptionCounts.get(page.description) ?? 0) + 1);
+    const generic = new Set([...descriptionCounts].filter(([, count]) => count >= GENERIC_DESCRIPTION_MIN_PAGES).map(([text]) => text));
+    const lines = [`# ${siteName ?? 'Site'}`];
+    if (summary)
+        lines.push('', `> ${summary}`);
+    let remaining = maxLinks;
+    const ordered = [...sections.entries()].sort(([a, pa], [b, pb]) => (a === 'Overview' ? -1 : 0) - (b === 'Overview' ? -1 : 0) || pb.length - pa.length || a.localeCompare(b));
+    for (const [name, pages] of ordered) {
+        if (remaining <= 0)
+            break;
+        // Flat sites keep most pages under Overview, so it gets a larger share than a topical section.
+        const limit = name === 'Overview' ? maxPerSection * OVERVIEW_LIMIT_FACTOR : maxPerSection;
+        const picked = [...pages]
+            .sort((a, b) => segmentsOf(a).length - segmentsOf(b).length || a.path.localeCompare(b.path))
+            .slice(0, Math.min(limit, remaining));
+        remaining -= picked.length;
+        lines.push('', `## ${name}`, '', ...picked.map((page) => linkLine(page, siteName, generic)));
+    }
+    const alternates = all
+        .filter((page) => lang !== undefined && page.lang && !sameLanguage(page.lang, lang) && segmentsOf(page).length === 0)
+        .sort((a, b) => (a.lang ?? '').localeCompare(b.lang ?? ''));
+    if (alternates.length > 0) {
+        lines.push('', '## Optional', '', ...alternates.map((page) => `- [${page.lang}](${page.url})`));
+    }
+    return `${lines.join('\n')}\n`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hagicode/hagilight",
-  "version": "0.9.5-dev.109.1.aafdacc",
+  "version": "0.9.7-dev.111.1.899dd08",
   "type": "module",
   "description": "Astro integrations and components for HagiCode sites",
   "repository": {
@@ -40,7 +40,7 @@
   },
   "dependencies": {
     "@astrojs/sitemap": "^3.7.4",
-    "@hagicode/hagilight-core": "0.9.5-dev.109.1.aafdacc"
+    "@hagicode/hagilight-core": "0.9.7-dev.111.1.899dd08"
   },
   "keywords": [
     "astro-component",