@janga/norna 0.7.18 → 0.7.23

package/scripts/lib/site-link-graph.mjs

@@ -0,0 +1,326 @@
+import { readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import {
+	createPageAliasModel,
+} from './page-aliases.mjs';
+import { parsePageMarkdownSource } from './page-markdown.mjs';
+import {
+	parseContentFrontmatter,
+	splitSiteFile,
+	toPosixPath,
+} from './site-content.mjs';
+import {
+	sitePagesLabel,
+	sitePublicDir,
+	sitePublicLabel,
+} from './site-paths.mjs';
+import { getSiteNodePathname } from './site-page-urls.mjs';
+import { getSiteStructure } from './site-structure.mjs';
+
+const internalUrlOrigin = 'https://norna.invalid';
+const externalSchemePattern = /^[a-z][a-z0-9+.-]*:/i;
+
+const readPublicFiles = async (directory, relativeDirectory = '') => {
+	const entries = await readdir(directory, { withFileTypes: true }).catch((error) => {
+		if (error?.code === 'ENOENT') return [];
+		throw error;
+	});
+	const files = [];
+
+	for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name, 'en'))) {
+		const relativePath = relativeDirectory
+			? path.join(relativeDirectory, entry.name)
+			: entry.name;
+		const entryPath = path.join(directory, entry.name);
+		if (entry.isDirectory()) {
+			files.push(...await readPublicFiles(entryPath, relativePath));
+		} else if (entry.isFile()) {
+			files.push({
+				filePath: entryPath,
+				label: `${sitePublicLabel}/${toPosixPath(relativePath)}`,
+				pathname: `/${toPosixPath(relativePath)}`,
+			});
+		}
+	}
+
+	return files;
+};
+
+const decodePathname = (pathname) => {
+	const decodedSegments = pathname.split('/').map((segment) => decodeURIComponent(segment));
+	if (decodedSegments.some((segment) => segment.includes('/') || segment.includes('\\') || segment.includes('\0'))) {
+		throw new Error('Encoded slashes, backslashes, and null bytes are not valid internal paths.');
+	}
+	return decodedSegments.join('/');
+};
+
+const decodeFragment = (fragment) => decodeURIComponent(fragment);
+
+const getPageLookupPathname = (pathname) => {
+	if (pathname === '/' || pathname === '/index.html') return '/';
+	if (pathname.endsWith('/index.html')) return pathname.slice(0, -'index.html'.length);
+	if (pathname.endsWith('/')) return pathname;
+	const finalSegment = pathname.split('/').at(-1) ?? '';
+	return finalSegment.includes('.') ? null : `${pathname}/`;
+};
+
+const isExternalTarget = (target) => target.startsWith('//') || externalSchemePattern.test(target);
+
+export const resolveInternalTarget = (target, sourcePathname) => {
+	if (isExternalTarget(target)) return { kind: 'external' };
+
+	let url;
+	try {
+		url = new URL(target, `${internalUrlOrigin}${sourcePathname}`);
+	} catch {
+		return {
+			kind: 'invalid',
+			reason: 'The target is not a valid relative or site-relative URL.',
+		};
+	}
+
+	if (url.origin !== internalUrlOrigin) return { kind: 'external' };
+
+	try {
+		const pathname = decodePathname(url.pathname);
+		const fragment = url.hash.length > 1 ? decodeFragment(url.hash.slice(1)) : '';
+		return {
+			kind: 'internal',
+			fragment,
+			pageLookupPathname: getPageLookupPathname(pathname),
+			pathname,
+			query: url.search,
+		};
+	} catch (error) {
+		return {
+			kind: 'invalid',
+			reason: error instanceof Error ? error.message : String(error),
+		};
+	}
+};
+
+const createPageRecord = ({ contentFile, data = {}, document }) => {
+	const pathname = getSiteNodePathname(contentFile);
+	const anchors = new Map([['page-title', {
+		id: 'page-title',
+		line: document.pageTitle?.line ?? 1,
+		title: document.pageTitle?.title ?? 'Page title',
+	}]]);
+
+	for (const heading of document.headings) {
+		if ((heading.depth !== 2 && heading.depth !== 3) || !heading.id || anchors.has(heading.id)) continue;
+		anchors.set(heading.id, heading);
+	}
+
+	return {
+		aliases: data.page?.aliases ?? [],
+		anchors,
+		contentFile,
+		document,
+		pathname,
+		title: document.pageTitle?.title ?? contentFile.pageId,
+	};
+};
+
+const createIssue = (reference, code, message, fix) => ({
+	code,
+	fix,
+	message,
+	reference,
+	severity: 'error',
+});
+
+const getTargetKey = ({ fragment, pageLookupPathname, pathname }) => {
+	const identityPathname = pageLookupPathname ?? pathname;
+	return `${identityPathname}${fragment ? `#${fragment}` : ''}`;
+};
+
+const looksLikePublicFile = (pathname) => {
+	if (pathname.endsWith('/')) return false;
+	return (pathname.split('/').at(-1) ?? '').includes('.');
+};
+
+export const createSiteLinkGraph = ({ siteStructure, pageDocuments, publicFiles = [] }) => {
+	const documentsByDirectory = new Map(pageDocuments.map(({ contentFile, data, document }) => [
+		contentFile.pageDirectory,
+		{ contentFile, data, document },
+	]));
+	const pages = siteStructure.contentFiles.map((contentFile) => {
+		const pageDocument = documentsByDirectory.get(contentFile.pageDirectory);
+		if (!pageDocument) throw new Error(`No parsed Markdown document was provided for ${contentFile.contentLabel}.`);
+		return createPageRecord(pageDocument);
+	});
+	const pagesByPathname = new Map(pages.map((page) => [page.pathname, page]));
+	const categoriesByPathname = new Map(siteStructure.categories.map((category) => [
+		getSiteNodePathname(category),
+		{ ...category, pathname: getSiteNodePathname(category) },
+	]));
+	const publicFilesByPathname = new Map();
+	for (const file of publicFiles) {
+		publicFilesByPathname.set(file.pathname, file);
+		if (file.pathname === '/index.html') publicFilesByPathname.set('/', file);
+		if (file.pathname.endsWith('/index.html')) {
+			publicFilesByPathname.set(file.pathname.slice(0, -'index.html'.length), file);
+		}
+	}
+	const aliasModel = createPageAliasModel({
+		categories: siteStructure.categories,
+		pages,
+		publicFiles,
+	});
+
+	const references = [];
+	const referencesByTarget = new Map();
+	const diagnostics = [...aliasModel.diagnostics];
+
+	for (const page of pages) {
+		for (const sourceReference of page.document.links) {
+			const target = resolveInternalTarget(sourceReference.target, page.pathname);
+			if (target.kind === 'external') continue;
+
+			const reference = {
+				...sourceReference,
+				sourceContentFile: page.contentFile,
+				sourcePage: page,
+				target,
+			};
+			references.push(reference);
+
+			if (target.kind === 'invalid') {
+				diagnostics.push(createIssue(
+					reference,
+					'invalid-internal-url',
+					`Internal link "${sourceReference.target}" on line ${sourceReference.line} is invalid. ${target.reason}`,
+					'Use a valid fragment, relative URL, site-relative URL, or external URL.',
+				));
+				continue;
+			}
+
+			const targetKey = getTargetKey(target);
+			if (!referencesByTarget.has(targetKey)) referencesByTarget.set(targetKey, []);
+			referencesByTarget.get(targetKey).push(reference);
+
+			const targetPage = target.pageLookupPathname
+				? pagesByPathname.get(target.pageLookupPathname)
+				: null;
+			if (targetPage) {
+				reference.resolution = { kind: 'page', page: targetPage, pathname: targetPage.pathname };
+				if (target.fragment && !targetPage.anchors.has(target.fragment)) {
+					const availableAnchors = Array.from(targetPage.anchors.keys());
+					const displayedAnchors = availableAnchors.slice(0, 8).map((anchor) => `#${anchor}`);
+					const omittedCount = availableAnchors.length - displayedAnchors.length;
+					diagnostics.push(createIssue(
+						reference,
+						'missing-internal-anchor',
+						`Internal link "${sourceReference.target}" on line ${sourceReference.line} points to missing heading anchor "#${target.fragment}" on ${targetPage.pathname}.`,
+						availableAnchors.length > 0
+							? `Use an existing anchor: ${displayedAnchors.join(', ')}${omittedCount > 0 ? `, and ${omittedCount} more` : ''}.`
+							: 'Add the intended H2 or H3 heading, or remove the fragment from the link.',
+					));
+				}
+				continue;
+			}
+
+			const targetAlias = target.pageLookupPathname
+				? aliasModel.aliasesByPathname.get(target.pageLookupPathname)
+				: null;
+			if (targetAlias) {
+				const aliasTargetPage = pagesByPathname.get(targetAlias.targetPathname);
+				if (!aliasTargetPage) {
+					throw new Error(`Page alias "${targetAlias.pathname}" has no target page ${targetAlias.targetPathname}.`);
+				}
+				reference.resolution = {
+					alias: targetAlias,
+					kind: 'page-alias',
+					page: aliasTargetPage,
+					pathname: aliasTargetPage.pathname,
+				};
+				if (target.fragment && !aliasTargetPage.anchors.has(target.fragment)) {
+					const availableAnchors = Array.from(aliasTargetPage.anchors.keys());
+					diagnostics.push(createIssue(
+						reference,
+						'missing-internal-anchor',
+						`Internal link "${sourceReference.target}" on line ${sourceReference.line} points through alias ${targetAlias.pathname} to missing heading anchor "#${target.fragment}" on ${aliasTargetPage.pathname}.`,
+						availableAnchors.length > 0
+							? `Use an existing anchor: ${availableAnchors.slice(0, 8).map((anchor) => `#${anchor}`).join(', ')}.`
+							: 'Add the intended H2 or H3 heading, or remove the fragment from the link.',
+					));
+				}
+				continue;
+			}
+
+			const targetCategory = target.pageLookupPathname
+				? categoriesByPathname.get(target.pageLookupPathname)
+				: null;
+			if (targetCategory) {
+				reference.resolution = { category: targetCategory, kind: 'category', pathname: targetCategory.pathname };
+				diagnostics.push(createIssue(
+					reference,
+					'category-has-no-url',
+					`Internal link "${sourceReference.target}" on line ${sourceReference.line} points to navigation category ${targetCategory.pathname}, which does not have its own page.`,
+					'Link to one of the category pages, or replace category.yaml with content.md when the collection needs its own page.',
+				));
+				continue;
+			}
+
+			const publicFile = publicFilesByPathname.get(target.pathname);
+			if (publicFile) {
+				reference.resolution = { file: publicFile, kind: 'public-file', pathname: target.pathname };
+				continue;
+			}
+
+			if (looksLikePublicFile(target.pathname)) {
+				reference.resolution = { kind: 'missing-public-file', pathname: target.pathname };
+				diagnostics.push(createIssue(
+					reference,
+					'missing-public-file',
+					`Internal link "${sourceReference.target}" on line ${sourceReference.line} points to public file "${target.pathname}", but no matching file exists under ${sitePublicLabel}/.`,
+					`Add the file under ${sitePublicLabel}/ with the same relative path, or correct the link.`,
+				));
+				continue;
+			}
+
+			const missingPathname = target.pageLookupPathname ?? target.pathname;
+			reference.resolution = { kind: 'missing-page', pathname: missingPathname };
+			diagnostics.push(createIssue(
+				reference,
+				'missing-internal-page',
+				`Internal link "${sourceReference.target}" on line ${sourceReference.line} points to page "${missingPathname}", but that page does not exist.`,
+				`Correct the link or create the page at the intended position under ${sitePagesLabel}/.`,
+			));
+		}
+	}
+
+	return {
+		aliasModel,
+		aliases: aliasModel.aliases,
+		aliasesByPathname: aliasModel.aliasesByPathname,
+		categoriesByPathname,
+		diagnostics,
+		pages,
+		pagesByPathname,
+		publicFilesByPathname,
+		references,
+		referencesByTarget,
+	};
+};
+
+export const getSiteLinkGraph = async (options = {}) => {
+	const siteStructure = options.siteStructure ?? await getSiteStructure();
+	const [pageDocuments, publicFiles] = await Promise.all([
+		Promise.all(siteStructure.contentFiles.map(async (contentFile) => {
+			const source = await readFile(contentFile.contentPath, 'utf8');
+			const { frontmatterBody } = splitSiteFile(source, contentFile.contentLabel);
+			return {
+				contentFile,
+				data: parseContentFrontmatter(frontmatterBody, contentFile.contentLabel),
+				document: await parsePageMarkdownSource(source, { label: contentFile.contentLabel }),
+			};
+		})),
+		readPublicFiles(options.publicDir ?? sitePublicDir),
+	]);
+
+	return createSiteLinkGraph({ pageDocuments, publicFiles, siteStructure });
+};
+
+export const getSitePublicFiles = async (publicDir = sitePublicDir) => readPublicFiles(publicDir);

package/scripts/lib/site-page-urls.mjs

@@ -0,0 +1,10 @@
+export const getSiteNodePathname = (node) => node.isHome || !node.pagePath
+	? '/'
+	: `/${node.pagePath}/`;
+
+export const getAbsolutePageUrl = (siteUrl, pathname) => {
+	const baseUrl = new URL(siteUrl);
+	if (!baseUrl.pathname.endsWith('/')) baseUrl.pathname = `${baseUrl.pathname}/`;
+
+	return new URL(pathname === '/' ? '' : pathname.replace(/^\//, ''), baseUrl).href;
+};

package/scripts/lib/site-paths.mjs

@@ -100,7 +100,6 @@ const resolveSitePaths = () => {
 const resolvedSitePaths = resolveSitePaths();
 
 export const siteProjectRoot = resolvedSitePaths.siteProjectRoot;
-export const root = siteProjectRoot;
 export const siteDirectory = resolvedSitePaths.siteDirectory;
 
 if (!siteDirectory) {
@@ -130,13 +129,13 @@ export const sitePublicDir = path.join(siteDir, 'public');
 export const siteStateDir = path.join(siteDir, '.norna');
 export const astroPublicDir = path.join(siteStateDir, 'public');
 export const astroDistDir = path.join(siteProjectRoot, 'dist');
-export const astroCacheDir = path.join(siteProjectRoot, '.astro');
+export const astroRootDir = siteStateDir;
+export const astroCacheDir = path.join(astroRootDir, '.astro');
 export const generatedImagesDir = path.join(astroPublicDir, 'images', 'generated');
 export const originalImagesDir = path.join(astroPublicDir, 'images', 'original');
 export const generatedImagesManifestPath = path.join(siteStateDir, 'generated-images.json');
 
 export const engineRootLabel = getPathLabel(engineRoot);
-export const invocationRootLabel = getPathLabel(invocationRoot);
 export const siteProjectRootLabel = getPathLabel(siteProjectRoot);
 export const siteDirLabel = getPathLabel(siteDir);
 export const siteConfigLabel = getPathLabel(siteConfigPath);

package/scripts/lib/site-structure.mjs

@@ -0,0 +1,189 @@
+import { access, readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { categorySchema } from './schema-definitions.mjs';
+import { parsePageDirectoryPath } from './page-model.mjs';
+import {
+	homePageDirectory,
+	siteContentLabel,
+	siteDir,
+	siteDirLabel,
+	sitePagesDir,
+	sitePagesLabel,
+} from './site-paths.mjs';
+import { parseYamlConfig } from './yaml-config.mjs';
+
+const fileExists = async (filePath) => access(filePath).then(() => true, () => false);
+
+const readDirectory = async (directory) => readdir(directory, { withFileTypes: true }).catch((error) => {
+	if (error?.code === 'ENOENT') return [];
+	throw error;
+});
+
+const compareNodeMetadata = (left, right) => (
+	left.pageOrder - right.pageOrder
+	|| left.pageId.localeCompare(right.pageId, 'en')
+);
+
+const assertLegacyStructureIsAbsent = async () => {
+	const legacyContentPath = path.join(siteDir, 'content.md');
+	const legacyImagesPath = path.join(siteDir, 'images');
+	const legacyPaths = [];
+	if (await fileExists(legacyContentPath)) legacyPaths.push(`${siteDirLabel}/content.md`);
+	if (await fileExists(legacyImagesPath)) legacyPaths.push(`${siteDirLabel}/images`);
+	if (legacyPaths.length > 0) {
+		throw new Error([
+			`The old root-page structure is no longer supported: ${legacyPaths.join(', ')}.`,
+			`Move the homepage content to ${sitePagesLabel}/${homePageDirectory}/content.md and its images to ${sitePagesLabel}/${homePageDirectory}/images/.`,
+		].join('\n'));
+	}
+
+	if (await fileExists(path.join(siteDir, 'routes'))) {
+		throw new Error(`${siteDirLabel}/routes is no longer supported. Rename it to ${sitePagesLabel} and use NNN-page-id directory names.`);
+	}
+};
+
+const assertUniqueSiblings = (nodes, pagesLabel) => {
+	const byId = new Map();
+	const byOrder = new Map();
+
+	for (const node of nodes) {
+		const existingId = byId.get(node.pageId);
+		if (existingId) {
+			throw new Error([
+				`${pagesLabel} contains duplicate sibling id "${node.pageId}".`,
+				`- ${existingId.nodeLabel}`,
+				`- ${node.nodeLabel}`,
+			].join('\n'));
+		}
+		byId.set(node.pageId, node);
+
+		const existingOrder = byOrder.get(node.pageOrder);
+		if (existingOrder) {
+			throw new Error([
+				`${pagesLabel} contains duplicate sibling order "${String(node.pageOrder).padStart(3, '0')}".`,
+				`- ${existingOrder.nodeLabel}`,
+				`- ${node.nodeLabel}`,
+			].join('\n'));
+		}
+		byOrder.set(node.pageOrder, node);
+	}
+};
+
+const readCategory = async (categoryPath, categorySourceLabel) => {
+	const source = await readFile(categoryPath, 'utf8');
+	return parseYamlConfig(source, categorySourceLabel, { schema: categorySchema });
+};
+
+export const getSiteStructure = async () => {
+	await assertLegacyStructureIsAbsent();
+
+	const nodes = [];
+	const warnings = [];
+
+	const collectNodes = async (pagesDir, pagesLabel, parentNodeDirectory = '') => {
+		const entries = (await readDirectory(pagesDir))
+			.filter((entry) => entry.isDirectory())
+			.sort((left, right) => left.name.localeCompare(right.name, 'en'));
+		const siblingNodes = [];
+
+		for (const entry of entries) {
+			const pageDirectory = parentNodeDirectory
+				? `${parentNodeDirectory}/pages/${entry.name}`
+				: entry.name;
+			const nodeLabel = `${pagesLabel}/${entry.name}`;
+			const nodeDir = path.join(pagesDir, entry.name);
+			const contentPath = path.join(nodeDir, 'content.md');
+			const categoryPath = path.join(nodeDir, 'category.yaml');
+			const [hasContent, hasCategory] = await Promise.all([
+				fileExists(contentPath),
+				fileExists(categoryPath),
+			]);
+			const metadata = parsePageDirectoryPath(pageDirectory, nodeLabel);
+
+			if (hasContent === hasCategory) {
+				const problem = hasContent
+					? 'contains both content.md and category.yaml'
+					: 'contains neither content.md nor category.yaml';
+				throw new Error(`${nodeLabel} ${problem}. Keep exactly one: content.md for a page, or category.yaml for a navigation category.`);
+			}
+
+			const isHome = pageDirectory === homePageDirectory;
+			if (isHome && hasCategory) {
+				throw new Error(`${nodeLabel}/category.yaml is invalid. The homepage must be a page with content.md.`);
+			}
+
+			const node = {
+				...metadata,
+				isHome,
+				kind: hasCategory ? 'category' : 'page',
+				nodeDir,
+				nodeLabel,
+				pagePath: isHome ? '' : metadata.pagePath,
+				parentPagePath: isHome ? null : metadata.parentPagePath,
+			};
+
+			if (hasCategory) {
+				if (await fileExists(path.join(nodeDir, 'images'))) {
+					throw new Error(`${nodeLabel} is a navigation category and cannot contain images/. Use content.md when the collection needs editorial content or images.`);
+				}
+				const category = await readCategory(categoryPath, `${nodeLabel}/category.yaml`);
+				Object.assign(node, {
+					categorySourceLabel: `${nodeLabel}/category.yaml`,
+					categoryPath,
+					label: category.label,
+				});
+			} else {
+				Object.assign(node, {
+					contentLabel: `${nodeLabel}/content.md`,
+					contentPath,
+					imagesDir: path.join(nodeDir, 'images'),
+					imagesLabel: `${nodeLabel}/images`,
+				});
+			}
+
+			siblingNodes.push(node);
+		}
+
+		assertUniqueSiblings(siblingNodes, pagesLabel);
+
+		for (const node of siblingNodes.sort(compareNodeMetadata)) {
+			nodes.push(node);
+			const childPagesDir = path.join(node.nodeDir, 'pages');
+			const childDirectories = (await readDirectory(childPagesDir)).filter((entry) => entry.isDirectory());
+
+			if (node.isHome && childDirectories.length > 0) {
+				throw new Error([
+					`${node.nodeLabel} is the homepage and cannot contain child pages.`,
+					`Move these page directories beside ${homePageDirectory} under ${sitePagesLabel}/, or below another non-home page:`,
+					...childDirectories.map(({ name }) => `- ${node.nodeLabel}/pages/${name}`),
+				].join('\n'));
+			}
+
+			if (node.kind === 'category' && childDirectories.length === 0) {
+				warnings.push({
+					code: 'empty-category',
+					label: node.categorySourceLabel,
+					message: `${node.categorySourceLabel} defines an empty navigation category. Add at least one child page under ${node.nodeLabel}/pages/.`,
+				});
+			}
+
+			if (!node.isHome && childDirectories.length > 0) {
+				await collectNodes(childPagesDir, `${node.nodeLabel}/pages`, node.pageDirectory);
+			}
+		}
+	};
+
+	await collectNodes(sitePagesDir, sitePagesLabel);
+	if (!nodes.some(({ isHome }) => isHome)) {
+		throw new Error(`Homepage content is missing. Create ${siteContentLabel}.`);
+	}
+
+	return {
+		categories: nodes.filter(({ kind }) => kind === 'category'),
+		contentFiles: nodes.filter(({ kind }) => kind === 'page'),
+		nodes,
+		warnings,
+	};
+};
+
+export const getContentFiles = async () => (await getSiteStructure()).contentFiles;

package/scripts/lib/sitemap.mjs

@@ -0,0 +1,34 @@
+import { getAbsolutePageUrl, getSiteNodePathname } from './site-page-urls.mjs';
+
+export const sitemapFilename = 'sitemap.xml';
+
+const escapeXml = (value) => String(value)
+	.replaceAll('&', '&')
+	.replaceAll('<', '&lt;')
+	.replaceAll('>', '&gt;')
+	.replaceAll('"', '&quot;')
+	.replaceAll("'", '&apos;');
+
+const compareStrings = (left, right) => left < right ? -1 : left > right ? 1 : 0;
+
+export const getSitemapUrls = ({ siteStructure, siteUrl }) => (
+	siteStructure.contentFiles
+		.map((page) => getSiteNodePathname(page))
+		.sort(compareStrings)
+		.map((pathname) => getAbsolutePageUrl(siteUrl, pathname))
+);
+
+export const createSitemapXml = (options) => {
+	const urls = getSitemapUrls(options);
+	const lines = [
+		'<?xml version="1.0" encoding="UTF-8"?>',
+		'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+	];
+
+	for (const url of urls) {
+		lines.push('  <url>', `    <loc>${escapeXml(url)}</loc>`, '  </url>');
+	}
+
+	lines.push('</urlset>');
+	return `${lines.join('\n')}\n`;
+};