methanol 0.0.11 → 0.0.12

package/src/mdx.js

@@ -131,9 +131,7 @@ const resolvePageHeadAssets = (page) => {
 export const buildPageContext = ({ routePath, path, pageMeta, pagesContext, lazyPagesTree = false }) => {
 	const page = pageMeta
 	const language = pagesContext.getLanguageForRoute ? pagesContext.getLanguageForRoute(routePath) : null
-	const getSiblings = pagesContext.getSiblings
-		? () => pagesContext.getSiblings(routePath, page.path || path)
-		: null
+	const getSiblings = pagesContext.getSiblings ? () => pagesContext.getSiblings(routePath, page.path || path) : null
 	if (page && getSiblings && page.getSiblings !== getSiblings) {
 		page.getSiblings = getSiblings
 	}
@@ -215,7 +213,7 @@ const normalizeStarryNightConfig = (value) => {
 	}
 	if (typeof value !== 'object') return null
 	const { enabled, options, ...rest } = value
-	if (enabled === false) return { enabled: false, options: null }
+	if (enabled === false) return { enabled: false, options }
 	if (options && typeof options === 'object') {
 		return { enabled: true, options: { ...options } }
 	}
@@ -289,12 +287,17 @@ const resolveMdxConfigForPage = async (frontmatter) => {
 	return mdxConfig
 }
 
-export const compileMdx = async ({ content, path, ctx }) => {
-	const mdxConfig = await resolveMdxConfigForPage(ctx.page.frontmatter)
-	const runtimeFactory = mdxConfig.development ? JSXDevFactory : JSXFactory
+export const compileMdxSource = async ({ content, path, frontmatter }) => {
+	const mdxConfig = await resolveMdxConfigForPage(frontmatter)
 	const compiled = await compile({ value: content, path: path }, mdxConfig)
+	const code = String(compiled.value ?? compiled)
+	return { code, development: Boolean(mdxConfig.development) }
+}
 
-	return await run(compiled, {
+export const runMdxSource = async ({ code, path, ctx, development = null }) => {
+	const isDev = development == null ? state.CURRENT_MODE !== 'production' : development
+	const runtimeFactory = isDev ? JSXDevFactory : JSXFactory
+	return await run(code, {
 		...runtimeFactory,
 		baseUrl: pathToFileURL(path).href,
 		ctx,
@@ -302,9 +305,23 @@ export const compileMdx = async ({ content, path, ctx }) => {
 	})
 }
 
+export const compileMdx = async ({ content, path, ctx }) => {
+	const result = await compileMdxSource({
+		content,
+		path,
+		frontmatter: ctx?.page?.frontmatter || null
+	})
+	return await runMdxSource({
+		code: result.code,
+		path,
+		ctx,
+		development: result.development
+	})
+}
+
 export const compilePageMdx = async (page, pagesContext, options = {}) => {
 	if (!page || page.content == null || page.mdxComponent) return
-	const { ctx = null, lazyPagesTree = false, refreshPagesTree = true } = options || {}
+	const { ctx = null, lazyPagesTree = false, refreshPagesTree = true, compiled = null } = options || {}
 	const activeCtx =
 		ctx ||
 		buildPageContext({
@@ -314,11 +331,18 @@ export const compilePageMdx = async (page, pagesContext, options = {}) => {
 			pagesContext,
 			lazyPagesTree
 		})
-	const mdxModule = await compileMdx({
-		content: page.content,
-		path: page.path,
-		ctx: activeCtx
-	})
+	const mdxModule = compiled?.code
+		? await runMdxSource({
+				code: compiled.code,
+				path: page.path,
+				ctx: activeCtx,
+				development: compiled.development
+			})
+		: await compileMdx({
+				content: page.content,
+				path: page.path,
+				ctx: activeCtx
+			})
 	page.mdxComponent = mdxModule.default
 	page.toc = mdxModule.toc
 	const shouldUseTocTitle = page.frontmatter?.title == null
@@ -346,41 +370,46 @@ export const renderHtml = async ({ routePath, path, components, pagesContext, pa
 		pageMeta,
 		pagesContext
 	})
-	await compilePageMdx(pageMeta, pagesContext, { ctx })
 
-	const [Head, Outlet] = createPortal()
-	const ExtraHead = () => {
-		return [
-			resolveRewindInject(),
-			...resolveUserHeadAssets(),
-			...resolvePageHeadAssets(pageMeta),
-			Outlet(),
-			RWND_FALLBACK
-		]
-	}
+	await compilePageMdx(pageMeta, pagesContext, { ctx })
 
+	const template = state.USER_THEME.template
 	const mdxComponent = pageMeta.mdxComponent
 
-	const PageContent = ({ components: extraComponents, ...props }, ...children) =>
-		mdxComponent({
-			children,
-			...props,
-			components: {
-				...components,
-				...extraComponents,
-				head: Head,
-				Head
-			}
-		})
+	const renderResult = await new Promise((resolve, reject) => {
+		const [Head, Outlet] = createPortal()
+		const ExtraHead = () => {
+			return [
+				resolveRewindInject(),
+				...resolveUserHeadAssets(),
+				...resolvePageHeadAssets(pageMeta),
+				Outlet(),
+				RWND_FALLBACK
+			]
+		}
 
-	const template = state.USER_THEME.template
+		const PageContent = ({ components: extraComponents, ...props }, ...children) => {
+			try {
+				return mdxComponent({
+					children,
+					...props,
+					components: {
+						...components,
+						...extraComponents,
+						head: Head,
+						Head
+					}
+				})
+			} catch (e) {
+				reject(e)
+			}
+		}
 
-	const renderResult = await new Promise((r) => {
 		const result = HTMLRenderer.c(
 			Suspense,
 			{
 				onLoad() {
-					nextTick(() => r(result))
+					nextTick(() => resolve(result))
 				}
 			},
 			() =>

package/src/pages.js

@@ -22,9 +22,11 @@ import matter from 'gray-matter'
 import { readdir, readFile, stat } from 'fs/promises'
 import { existsSync } from 'fs'
 import { resolve, join, relative } from 'path'
+import { cpus } from 'os'
+import { Worker } from 'worker_threads'
 import { state, cli } from './state.js'
 import { withBase } from './config.js'
-import { compilePageMdx } from './mdx.js'
+import { compileMdxSource, compilePageMdx } from './mdx.js'
 import { createStageLogger } from './stage-logger.js'
 
 const isPageFile = (name) => name.endsWith('.mdx') || name.endsWith('.md')
@@ -32,6 +34,137 @@ const isIgnoredEntry = (name) => name.startsWith('.') || name.startsWith('_')
 
 const pageMetadataCache = new Map()
 const pageDerivedCache = new Map()
+const MDX_WORKER_URL = new URL('./workers/mdx-compile-worker.js', import.meta.url)
+const cliOverrides = {
+	CLI_INTERMEDIATE_DIR: cli.CLI_INTERMEDIATE_DIR,
+	CLI_EMIT_INTERMEDIATE: cli.CLI_EMIT_INTERMEDIATE,
+	CLI_HOST: cli.CLI_HOST,
+	CLI_PORT: cli.CLI_PORT,
+	CLI_PAGES_DIR: cli.CLI_PAGES_DIR,
+	CLI_COMPONENTS_DIR: cli.CLI_COMPONENTS_DIR,
+	CLI_ASSETS_DIR: cli.CLI_ASSETS_DIR,
+	CLI_OUTPUT_DIR: cli.CLI_OUTPUT_DIR,
+	CLI_CONFIG_PATH: cli.CLI_CONFIG_PATH,
+	CLI_SITE_NAME: cli.CLI_SITE_NAME,
+	CLI_CODE_HIGHLIGHTING: cli.CLI_CODE_HIGHLIGHTING,
+	CLI_JOBS: cli.CLI_JOBS,
+	CLI_VERBOSE: cli.CLI_VERBOSE,
+	CLI_BASE: cli.CLI_BASE,
+	CLI_SEARCH: cli.CLI_SEARCH,
+	CLI_PWA: cli.CLI_PWA
+}
+
+const resolveWorkerCount = (total) => {
+	const cpuCount = Math.max(1, cpus()?.length || 1)
+	const requested = state.WORKER_JOBS
+	if (requested == null || requested <= 0) {
+		const items = Math.max(1, Number.isFinite(total) ? total : 1)
+		const autoCount = Math.round(Math.log(items))
+		return Math.max(1, Math.min(cpuCount, autoCount))
+	}
+	return Math.max(1, Math.min(cpuCount, Math.floor(requested)))
+}
+
+const compileMdxSources = async (pages, options = {}) => {
+	const targets = pages.filter((page) => page && page.content != null && !page.mdxComponent)
+	const results = new Map()
+	if (!targets.length) return results
+	const { onProgress } = options || {}
+	const reportProgress = (page) => {
+		if (typeof onProgress === 'function') {
+			onProgress(page)
+		}
+	}
+	const workerCount = Math.min(resolveWorkerCount(targets.length), targets.length)
+	if (workerCount <= 1) {
+		for (const page of targets) {
+			const result = await compileMdxSource({
+				content: page.content,
+				path: page.path,
+				frontmatter: page.frontmatter
+			})
+			results.set(page, result)
+			reportProgress(page)
+		}
+		return results
+	}
+
+	return await new Promise((resolve, reject) => {
+		const workers = []
+		const pending = new Map()
+		let cursor = 0
+		let nextId = 0
+		let finished = false
+
+		const finalize = async (error) => {
+			if (finished) return
+			finished = true
+			await Promise.all(workers.map((worker) => worker.terminate().catch(() => null)))
+			if (error) {
+				reject(error)
+				return
+			}
+			resolve(results)
+		}
+
+		const assign = (worker) => {
+			if (cursor >= targets.length) return false
+			const page = targets[cursor++]
+			const id = nextId++
+			pending.set(id, page)
+			worker.postMessage({
+				id,
+				path: page.path,
+				content: page.content,
+				frontmatter: page.frontmatter
+			})
+			return true
+		}
+
+		const handleMessage = (worker, message) => {
+			if (finished) return
+			const { id, result, error } = message || {}
+			const page = pending.get(id)
+			pending.delete(id)
+			if (!page) return
+			if (error) {
+				void finalize(new Error(error))
+				return
+			}
+			results.set(page, result)
+			reportProgress(page)
+			assign(worker)
+			if (results.size === targets.length && pending.size === 0) {
+				void finalize()
+			}
+		}
+
+		const handleError = (error) => {
+			if (finished) return
+			void finalize(error instanceof Error ? error : new Error(String(error)))
+		}
+
+		for (let i = 0; i < workerCount; i += 1) {
+			const worker = new Worker(MDX_WORKER_URL, {
+				type: 'module',
+				workerData: {
+					mode: state.CURRENT_MODE,
+					configPath: cli.CLI_CONFIG_PATH,
+					cli: cliOverrides
+				}
+			})
+			workers.push(worker)
+			worker.on('message', (message) => handleMessage(worker, message))
+			worker.on('error', handleError)
+			worker.on('exit', (code) => {
+				if (code !== 0) {
+					handleError(new Error(`MDX worker exited with code ${code}`))
+				}
+			})
+			assign(worker)
+		}
+	})
+}
 
 const collectLanguagesFromPages = (pages = []) => {
 	const languages = new Map()
@@ -616,47 +749,12 @@ const buildNavSequence = (nodes, pagesByRoute) => {
 	return result
 }
 
-export const buildPagesContext = async ({ compileAll = true } = {}) => {
-	const logEnabled = state.CURRENT_MODE === 'production' && cli.command === 'build'
-	const stageLogger = createStageLogger(logEnabled)
-	const collectToken = stageLogger.start('Collecting pages')
-	const collected = await collectPages()
-	stageLogger.end(collectToken)
-	let pages = collected.pages
-	const excludedRoutes = collected.excludedRoutes
-	const excludedDirs = collected.excludedDirs
-	const hasIndex = pages.some((page) => page.routePath === '/')
-	if (!hasIndex) {
-		const content = buildIndexFallback(pages, state.SITE_NAME)
-		pages.unshift({
-			routePath: '/',
-			routeHref: withBase('/'),
-			path: resolve(state.PAGES_DIR, 'index.md'),
-			relativePath: 'index.md',
-			name: 'index',
-			dir: '',
-			segments: [],
-			depth: 0,
-			isIndex: true,
-			title: state.SITE_NAME || 'Methanol Site',
-			weight: null,
-			date: null,
-			isRoot: false,
-			hidden: false,
-			content,
-			frontmatter: {},
-			matter: null,
-			stats: { size: content.length, createdAt: null, updatedAt: null },
-			createdAt: null,
-			updatedAt: null
-		})
-		if (excludedRoutes?.has('/')) {
-			excludedRoutes.delete('/')
-		}
-	}
-
+export const createPagesContextFromPages = ({ pages, excludedRoutes, excludedDirs } = {}) => {
+	const pageList = Array.isArray(pages) ? pages : []
+	const routeExcludes = excludedRoutes || new Set()
+	const dirExcludes = excludedDirs || new Set()
 	const pagesByRoute = new Map()
-	for (const page of pages) {
+	for (const page of pageList) {
 		if (!pagesByRoute.has(page.routePath)) {
 			pagesByRoute.set(page.routePath, page)
 		}
@@ -703,7 +801,7 @@ export const buildPagesContext = async ({ compileAll = true } = {}) => {
 	}
 	let pagesTree = getPagesTree('/')
 	const notFound = pagesByRoute.get('/404') || null
-	const languages = collectLanguagesFromPages(pages)
+	const languages = collectLanguagesFromPages(pageList)
 	const userSite = state.USER_SITE || {}
 	const siteBase = state.VITE_BASE ?? userSite.base ?? null
 	const site = {
@@ -723,9 +821,9 @@ export const buildPagesContext = async ({ compileAll = true } = {}) => {
 		},
 		generatedAt: new Date().toISOString()
 	}
-	const excludedDirPaths = new Set(Array.from(excludedDirs).map((dir) => `/${dir}`))
+	const excludedDirPaths = new Set(Array.from(dirExcludes).map((dir) => `/${dir}`))
 	const pagesContext = {
-		pages,
+		pages: pageList,
 		pagesByRoute,
 		getPageByRoute,
 		pagesTree,
@@ -775,34 +873,96 @@ export const buildPagesContext = async ({ compileAll = true } = {}) => {
 			pagesContext.getLanguageForRoute = (routePath) =>
 				resolveLanguageForRoute(pagesContext.languages, routePath)
 		},
-		excludedRoutes,
-		excludedDirs,
+		excludedRoutes: routeExcludes,
+		excludedDirs: dirExcludes,
 		excludedDirPaths,
 		notFound,
 		languages,
 		getLanguageForRoute: (routePath) => resolveLanguageForRoute(languages, routePath),
 		site
 	}
+	return pagesContext
+}
+
+export const buildPagesContext = async ({ compileAll = true } = {}) => {
+	const logEnabled = state.CURRENT_MODE === 'production' && cli.command === 'build'
+	const stageLogger = createStageLogger(logEnabled)
+	const collectToken = stageLogger.start('Collecting pages')
+	const collected = await collectPages()
+	stageLogger.end(collectToken)
+	let pages = collected.pages
+	const excludedRoutes = collected.excludedRoutes
+	const excludedDirs = collected.excludedDirs
+	const hasIndex = pages.some((page) => page.routePath === '/')
+	if (!hasIndex) {
+		const content = buildIndexFallback(pages, state.SITE_NAME)
+		pages.unshift({
+			routePath: '/',
+			routeHref: withBase('/'),
+			path: resolve(state.PAGES_DIR, 'index.md'),
+			relativePath: 'index.md',
+			name: 'index',
+			dir: '',
+			segments: [],
+			depth: 0,
+			isIndex: true,
+			title: state.SITE_NAME || 'Methanol Site',
+			weight: null,
+			date: null,
+			isRoot: false,
+			hidden: false,
+			content,
+			frontmatter: {},
+			matter: null,
+			stats: { size: content.length, createdAt: null, updatedAt: null },
+			createdAt: null,
+			updatedAt: null
+		})
+		if (excludedRoutes?.has('/')) {
+			excludedRoutes.delete('/')
+		}
+	}
+
+	const pagesContext = createPagesContextFromPages({
+		pages,
+		excludedRoutes,
+		excludedDirs
+	})
 	if (compileAll) {
 		const compileToken = stageLogger.start('Compiling MDX')
-		const totalPages = pages.length
-		for (let i = 0; i < pages.length; i++) {
-			const page = pages[i]
-			if (logEnabled) {
+		const compileTargets = pages.filter((page) => page && page.content != null && !page.mdxComponent)
+		const totalPages = compileTargets.length
+		let completed = 0
+		const compiledSources = await compileMdxSources(compileTargets, {
+			onProgress: (page) => {
+				if (!logEnabled) return
+				completed += 1
 				stageLogger.update(
 					compileToken,
-					`Compiling MDX [${i + 1}/${totalPages}] ${page.routePath || page.path}`
+					`Compiling MDX [${completed}/${totalPages}] ${page.routePath || page.path}`
 				)
 			}
+		})
+		stageLogger.end(compileToken)
+		const executeToken = stageLogger.start('Running MDX')
+		completed = 0
+		for (const page of compileTargets) {
+			const compiled = compiledSources.get(page) || null
 			await compilePageMdx(page, pagesContext, {
 				lazyPagesTree: true,
-				refreshPagesTree: false
+				refreshPagesTree: false,
+				compiled
 			})
+			if (logEnabled) {
+				completed += 1
+				stageLogger.update(
+					executeToken,
+					`Running MDX [${completed}/${totalPages}] ${page.routePath || page.path}`
+				)
+			}
 		}
-		stageLogger.end(compileToken)
-		pagesTreeCache.clear()
-		pagesTree = getPagesTree('/')
-		pagesContext.pagesTree = pagesTree
+		stageLogger.end(executeToken)
+		pagesContext.refreshPagesTree?.()
 	}
 	return pagesContext
 }

package/src/state.js

@@ -104,14 +104,14 @@ const withCommonOptions = (y) =>
 		.option('highlight', {
 			describe: 'Enable or disable code highlighting',
 			type: 'boolean',
-			coerce: (value) => {
-				if (value == null) return null
-				if (typeof value === 'boolean') return value
-				const normalized = String(value).trim().toLowerCase()
-				if (normalized === 'true') return true
-				if (normalized === 'false') return false
-				return null
-			}
+			default: undefined
+		})
+		.option('jobs', {
+			alias: 'j',
+			describe: 'Worker thread count (0 for auto)',
+			type: 'number',
+			requiresArg: true,
+			nargs: 1
 		})
 		.option('verbose', {
 			alias: 'v',
@@ -126,12 +126,14 @@ const withCommonOptions = (y) =>
 			nargs: 1
 		})
 		.option('search', {
-			describe: 'Enable search indexing (pagefind)',
-			type: 'boolean'
+			describe: 'Enable or disable search indexing (pagefind)',
+			type: 'boolean',
+			default: undefined
 		})
 		.option('pwa', {
-			describe: 'Enable PWA support',
-			type: 'boolean'
+			describe: 'Enable or disable PWA support',
+			type: 'boolean',
+			default: undefined
 		})
 
 const parser = yargs(hideBin(process.argv))
@@ -161,10 +163,11 @@ export const cli = {
 	CLI_CONFIG_PATH: argv.config || null,
 	CLI_SITE_NAME: argv['site-name'] || null,
 	CLI_CODE_HIGHLIGHTING: typeof argv.highlight === 'boolean' ? argv.highlight : null,
+	CLI_JOBS: typeof argv.jobs === 'number' && Number.isFinite(argv.jobs) ? argv.jobs : null,
 	CLI_VERBOSE: Boolean(argv.verbose),
 	CLI_BASE: argv.base || null,
-	CLI_SEARCH: argv.search,
-	CLI_PWA: argv.pwa
+	CLI_SEARCH: typeof argv.search === 'boolean' ? argv.search : undefined,
+	CLI_PWA: typeof argv.pwa === 'boolean' ? argv.pwa : undefined
 }
 
 export const state = {
@@ -204,6 +207,7 @@ export const state = {
 	THEME_POST_BUNDLE_HOOKS: [],
 	STARRY_NIGHT_ENABLED: false,
 	STARRY_NIGHT_OPTIONS: null,
+	WORKER_JOBS: 0,
 	GFM_ENABLED: true,
 	PWA_ENABLED: false,
 	PWA_OPTIONS: null,