methanol 0.0.10 → 0.0.12

package/README.md

@@ -41,9 +41,12 @@ export default () => ({
 		enabled: true
 	},
 
-	// optional: code highlighting (Starry Night)
+	// optional: code highlighting (Starry Night, default: enabled)
 	starryNight: false,
 
+	// optional: worker thread count (0 = auto)
+	jobs: 0,
+
 	// optional: pwa support
 	pwa: true,
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "methanol",
-  "version": "0.0.10",
+  "version": "0.0.12",
   "description": "Static site generator powered by rEFui and MDX",
   "main": "./index.js",
   "type": "module",

package/src/build-system.js

@@ -26,8 +26,8 @@ import { VitePWA } from 'vite-plugin-pwa'
 import { state, cli } from './state.js'
 import { resolveUserViteConfig } from './config.js'
 import { buildPagesContext } from './pages.js'
-import { renderHtml } from './mdx.js'
 import { buildComponentRegistry } from './components.js'
+import { createBuildWorkers, runWorkerStage, terminateWorkers } from './workers/build-pool.js'
 import { methanolVirtualHtmlPlugin, methanolResolverPlugin } from './vite-plugins.js'
 import { createStageLogger } from './stage-logger.js'
 import { preparePublicAssets } from './public-assets.js'
@@ -40,7 +40,6 @@ const ensureDir = async (dir) => {
 }
 
 const isHtmlFile = (name) => name.endsWith('.html')
-
 const collectHtmlFiles = async (dir, basePath = '') => {
 	const entries = await readdir(dir)
 	const files = []
@@ -76,22 +75,14 @@ export const buildHtmlEntries = async () => {
 	const stageLogger = createStageLogger(logEnabled)
 	const themeComponentsDir = state.THEME_COMPONENTS_DIR
 	const themeEnv = state.THEME_ENV
-	const themeRegistry = themeComponentsDir
-		? await buildComponentRegistry({
-				componentsDir: themeComponentsDir,
-				client: themeEnv.client
-			})
-		: { components: {} }
-	const themeComponents = {
-		...(themeRegistry.components || {}),
-		...(state.USER_THEME.components || {})
-	}
-	const { components } = await buildComponentRegistry()
-	const mergedComponents = {
-		...themeComponents,
-		...components
+	if (themeComponentsDir) {
+		await buildComponentRegistry({
+			componentsDir: themeComponentsDir,
+			client: themeEnv.client
+		})
 	}
-	const pagesContext = await buildPagesContext()
+	await buildComponentRegistry()
+	const pagesContext = await buildPagesContext({ compileAll: false })
 	const entry = {}
 	const htmlCache = new Map()
 	const resolveOutputName = (page) => {
@@ -102,39 +93,123 @@ export const buildHtmlEntries = async () => {
 		return page.routePath.slice(1)
 	}
 
-	const renderToken = stageLogger.start('Rendering pages')
-	const totalPages = pagesContext.pages.length
-	for (let i = 0; i < pagesContext.pages.length; i++) {
-		const page = pagesContext.pages[i]
-		if (logEnabled) {
-			stageLogger.update(renderToken, `Rendering pages [${i + 1}/${totalPages}] ${page.path}`)
+	const pages = pagesContext.pages || []
+	const totalPages = pages.length
+	const { workers, assignments } = createBuildWorkers(totalPages)
+	const excludedRoutes = Array.from(pagesContext.excludedRoutes || [])
+	const excludedDirs = Array.from(pagesContext.excludedDirs || [])
+	try {
+		await runWorkerStage({
+			workers,
+			stage: 'setPages',
+			messages: workers.map((worker) => ({
+				worker,
+				message: {
+					type: 'setPages',
+					stage: 'setPages',
+					pages,
+					excludedRoutes,
+					excludedDirs
+				}
+			}))
+		})
+
+		const compileToken = stageLogger.start('Compiling MDX')
+		let completed = 0
+		const updates = await runWorkerStage({
+			workers,
+			stage: 'compile',
+			messages: workers.map((worker, index) => ({
+				worker,
+				message: {
+					type: 'compile',
+					stage: 'compile',
+					ids: assignments[index]
+				}
+			})),
+			onProgress: (count) => {
+				if (!logEnabled) return
+				completed = count
+				stageLogger.update(compileToken, `Compiling MDX [${completed}/${totalPages}]`)
+			},
+			collect: (message) => message.updates || []
+		})
+		stageLogger.end(compileToken)
+
+		for (const update of updates) {
+			const page = pages[update.id]
+			if (!page) continue
+			if (update.title !== undefined) page.title = update.title
+			if (update.toc !== undefined) page.toc = update.toc
+			if (typeof pagesContext.setDerivedTitle === 'function') {
+				const shouldUseTocTitle = page.frontmatter?.title == null
+				pagesContext.setDerivedTitle(page.path, shouldUseTocTitle ? page.title : null, page.toc || null)
+			}
 		}
-		const html = await renderHtml({
-			routePath: page.routePath,
-			path: page.path,
-			components: mergedComponents,
-			pagesContext,
-			pageMeta: page
+		pagesContext.refreshPagesTree?.()
+
+		const titleSnapshot = pages.map((page) => page.title)
+		await runWorkerStage({
+			workers,
+			stage: 'sync',
+			messages: workers.map((worker) => ({
+				worker,
+				message: {
+					type: 'sync',
+					stage: 'sync',
+					updates,
+					titles: titleSnapshot
+				}
+			}))
 		})
-		const name = resolveOutputName(page)
-		const id = normalizePath(resolve(state.VIRTUAL_HTML_OUTPUT_ROOT, `${name}.html`))
-		entry[name] = id
-		htmlCache.set(id, html)
-		if (state.INTERMEDIATE_DIR) {
-			const outPath = resolve(state.INTERMEDIATE_DIR, `${name}.html`)
-			await ensureDir(dirname(outPath))
-			await writeFile(outPath, html)
+
+		const renderToken = stageLogger.start('Rendering pages')
+		completed = 0
+		const rendered = await runWorkerStage({
+			workers,
+			stage: 'render',
+			messages: workers.map((worker, index) => ({
+				worker,
+				message: {
+					type: 'render',
+					stage: 'render',
+					ids: assignments[index]
+				}
+			})),
+			onProgress: (count) => {
+				if (!logEnabled) return
+				completed = count
+				stageLogger.update(renderToken, `Rendering pages [${completed}/${totalPages}]`)
+			},
+			collect: (message) => message.results || []
+		})
+		stageLogger.end(renderToken)
+
+		for (const item of rendered) {
+			const page = pages[item.id]
+			if (!page) continue
+			const html = item.html
+			const name = resolveOutputName(page)
+			const id = normalizePath(resolve(state.VIRTUAL_HTML_OUTPUT_ROOT, `${name}.html`))
+			entry[name] = id
+			htmlCache.set(id, html)
+			if (state.INTERMEDIATE_DIR) {
+				const outPath = resolve(state.INTERMEDIATE_DIR, `${name}.html`)
+				await ensureDir(dirname(outPath))
+				await writeFile(outPath, html)
+			}
 		}
+	} finally {
+		await terminateWorkers(workers)
 	}
-	stageLogger.end(renderToken)
 
 	const htmlFiles = await collectHtmlFiles(state.PAGES_DIR)
-	const excludedDirs = pagesContext.excludedDirs || new Set()
+	const htmlExcludedDirs = pagesContext.excludedDirs || new Set()
 	const isHtmlExcluded = (relativePath) => {
-		if (!excludedDirs.size) return false
+		if (!htmlExcludedDirs.size) return false
 		const dir = relativePath.split('/').slice(0, -1).join('/')
 		if (!dir) return false
-		for (const excludedDir of excludedDirs) {
+		for (const excludedDir of htmlExcludedDirs) {
 			if (!excludedDir) return true
 			if (dir === excludedDir || dir.startsWith(`${excludedDir}/`)) {
 				return true

package/src/config.js

@@ -191,12 +191,12 @@ const resolvePagefindBuild = (config) => {
 }
 
 const resolveStarryNightConfig = (value) => {
-	if (value == null) return { enabled: false, options: null }
+	if (value == null) return { enabled: true, options: null }
 	if (typeof value === 'boolean') {
 		return { enabled: value, options: null }
 	}
 	if (typeof value !== 'object') {
-		return { enabled: false, options: null }
+		return { enabled: true, options: null }
 	}
 	const { enabled, options, ...rest } = value
 	if (enabled === false) return { enabled: false, options: null }
@@ -216,6 +216,14 @@ const normalizeHooks = (value) => {
 	return []
 }
 
+const normalizeJobs = (value) => {
+	if (value == null) return null
+	const parsed = Number(value)
+	if (!Number.isFinite(parsed)) return null
+	if (parsed <= 0) return 0
+	return Math.floor(parsed)
+}
+
 const loadConfigModule = async (filePath) => {
 	return import(`${pathToFileURL(filePath).href}?t=${Date.now()}`)
 }
@@ -425,6 +433,9 @@ export const applyConfig = async (config, mode) => {
 		state.STARRY_NIGHT_OPTIONS = starryNight.enabled ? starryNight.options : null
 	}
 
+	const jobsValue = cli.CLI_JOBS != null ? cli.CLI_JOBS : config.jobs
+	state.WORKER_JOBS = normalizeJobs(jobsValue) ?? 0
+
 	if (cli.CLI_INTERMEDIATE_DIR) {
 		state.INTERMEDIATE_DIR = resolveFromRoot(root, cli.CLI_INTERMEDIATE_DIR, 'build')
 	} else if (config.intermediateDir) {

package/src/dev-server.js

@@ -40,6 +40,8 @@ import { buildPagesContext, buildPageEntry, routePathFromFile } from './pages.js
 import { compilePageMdx, renderHtml } from './mdx.js'
 import { methanolResolverPlugin } from './vite-plugins.js'
 import { preparePublicAssets, updateAsset } from './public-assets.js'
+import { createBuildWorkers, runWorkerStage, terminateWorkers } from './workers/build-pool.js'
+import { style } from './logger.js'
 
 export const runViteDev = async () => {
 	const baseFsAllow = [state.ROOT_DIR, state.USER_THEME.root].filter(Boolean)
@@ -152,19 +154,128 @@ export const runViteDev = async () => {
 		htmlCache.clear()
 	}
 
+	const logMdxError = (phase, error, page = null) => {
+		const target = page?.path || page?.routePath || 'unknown file'
+		console.error(style.red(`\n[methanol] ${phase} error in ${target}`))
+		console.error(error?.stack || error)
+	}
+
 	const refreshPagesContext = async () => {
 		setPagesContext(await buildPagesContext({ compileAll: false }))
 	}
 
+	const prebuildHtmlCache = async (token) => {
+		if (!pagesContext || token !== pagesContextToken) return
+		const pages = pagesContext.pages || []
+		if (!pages.length) return
+		const { workers, assignments } = createBuildWorkers(pages.length, { command: 'serve' })
+		const excludedRoutes = Array.from(pagesContext.excludedRoutes || [])
+		const excludedDirs = Array.from(pagesContext.excludedDirs || [])
+		try {
+			await runWorkerStage({
+				workers,
+				stage: 'setPages',
+				messages: workers.map((worker) => ({
+					worker,
+					message: {
+						type: 'setPages',
+						stage: 'setPages',
+						pages,
+						excludedRoutes,
+						excludedDirs
+					}
+				}))
+			})
+			if (token !== pagesContextToken) return
+
+			const updates = await runWorkerStage({
+				workers,
+				stage: 'compile',
+				messages: workers.map((worker, index) => ({
+					worker,
+					message: {
+						type: 'compile',
+						stage: 'compile',
+						ids: assignments[index]
+					}
+				})),
+				collect: (message) => message.updates || []
+			})
+			if (token !== pagesContextToken) return
+
+			for (const update of updates) {
+				const page = pages[update.id]
+				if (!page) continue
+				if (update.title !== undefined) page.title = update.title
+				if (update.toc !== undefined) page.toc = update.toc
+				if (typeof pagesContext.setDerivedTitle === 'function') {
+					const shouldUseTocTitle = page.frontmatter?.title == null
+					pagesContext.setDerivedTitle(page.path, shouldUseTocTitle ? page.title : null, page.toc || null)
+				}
+			}
+			pagesContext.refreshPagesTree?.()
+			invalidateHtmlCache()
+			const renderEpoch = htmlCacheEpoch
+
+			const titleSnapshot = pages.map((page) => page.title)
+			await runWorkerStage({
+				workers,
+				stage: 'sync',
+				messages: workers.map((worker) => ({
+					worker,
+					message: {
+						type: 'sync',
+						stage: 'sync',
+						updates,
+						titles: titleSnapshot
+					}
+				}))
+			})
+			if (token !== pagesContextToken) return
+
+			const rendered = await runWorkerStage({
+				workers,
+				stage: 'render',
+				messages: workers.map((worker, index) => ({
+					worker,
+					message: {
+						type: 'render',
+						stage: 'render',
+						ids: assignments[index]
+					}
+				})),
+				collect: (message) => message.results || []
+			})
+			if (token !== pagesContextToken || renderEpoch !== htmlCacheEpoch) return
+			for (const item of rendered) {
+				const page = pages[item.id]
+				if (!page) continue
+				htmlCache.set(page.routePath, {
+					html: item.html,
+					path: page.path,
+					epoch: renderEpoch,
+					token
+				})
+			}
+		} finally {
+			await terminateWorkers(workers)
+		}
+	}
+
 	const runInitialCompile = async () => {
 		const token = pagesContextToken
 		try {
-			const nextContext = await buildPagesContext({ compileAll: true })
+			const nextContext = await buildPagesContext({ compileAll: false })
 			if (token !== pagesContextToken) {
 				return
 			}
 			setPagesContext(nextContext)
+			const nextToken = pagesContextToken
 			invalidateHtmlCache()
+			await prebuildHtmlCache(nextToken)
+			if (nextToken !== pagesContextToken) {
+				return
+			}
 			reload()
 		} catch (err) {
 			console.error(err)
@@ -257,10 +368,6 @@ export const runViteDev = async () => {
 			return next()
 		}
 
-		if (pathname.includes('.') && !pathname.endsWith('.html')) {
-			return next()
-		}
-
 		const accept = req.headers.accept || ''
 		if (!pathname.endsWith('.html') && !accept.includes('text/html')) {
 			return next()
@@ -274,6 +381,11 @@ export const runViteDev = async () => {
 			}
 		}
 		const requestedPath = routePath
+		if (pathname.includes('.') && !pathname.endsWith('.html')) {
+			if (!pagesContext?.pagesByRoute?.has(requestedPath)) {
+				return next()
+			}
+		}
 		const isExcludedPath = () => {
 			const excludedRoutes = pagesContext.excludedRoutes
 			if (excludedRoutes?.has(requestedPath)) return true
@@ -347,7 +459,12 @@ export const runViteDev = async () => {
 		try {
 			const renderEpoch = htmlCacheEpoch
 			const cacheEntry = htmlCache.get(renderRoutePath)
-			if (cacheEntry && cacheEntry.path === path && cacheEntry.epoch === htmlCacheEpoch) {
+			if (
+				cacheEntry &&
+				cacheEntry.path === path &&
+				cacheEntry.epoch === htmlCacheEpoch &&
+				cacheEntry.token === pagesContextToken
+			) {
 				res.statusCode = status
 				res.setHeader('Content-Type', 'text/html')
 				res.end(cacheEntry.html)
@@ -356,28 +473,37 @@ export const runViteDev = async () => {
 
 			pageMeta ??= pagesContext.getPageByRoute(renderRoutePath, { path })
 
-			const html = await renderHtml({
-				routePath: renderRoutePath,
-				path,
-				components: {
-					...themeComponents,
-					...components
-				},
-				pagesContext,
-				pageMeta
-			})
+			let html = ''
+			try {
+				html = await renderHtml({
+					routePath: renderRoutePath,
+					path,
+					components: {
+						...themeComponents,
+						...components
+					},
+					pagesContext,
+					pageMeta
+				})
+			} catch (err) {
+				logMdxError('MDX render', err, pageMeta || { path, routePath: renderRoutePath })
+				res.statusCode = 500
+				res.end('Internal Server Error')
+				return
+			}
 			if (renderEpoch === htmlCacheEpoch) {
 				htmlCache.set(renderRoutePath, {
 					html,
 					path,
-					epoch: renderEpoch
+					epoch: renderEpoch,
+					token: pagesContextToken
 				})
 			}
 			res.statusCode = status
 			res.setHeader('Content-Type', 'text/html')
 			res.end(html)
 		} catch (err) {
-			console.error(err)
+			logMdxError('MDX render', err, pageMeta || { path, routePath: renderRoutePath })
 			res.statusCode = 500
 			res.end('Internal Server Error')
 		}
@@ -532,12 +658,17 @@ export const runViteDev = async () => {
 		pagesContext.refreshPagesTree?.()
 		pagesContext.refreshLanguages?.()
 		if (prevEntry.content && prevEntry.content.trim().length) {
-			await compilePageMdx(prevEntry, pagesContext, {
-				lazyPagesTree: true,
-				refreshPagesTree: false
-			})
-			// Avoid caching a potentially stale render; recompile on request.
-			prevEntry.mdxComponent = null
+			try {
+				await compilePageMdx(prevEntry, pagesContext, {
+					lazyPagesTree: true,
+					refreshPagesTree: false
+				})
+				// Avoid caching a potentially stale render; recompile on request.
+				prevEntry.mdxComponent = null
+			} catch (err) {
+				logMdxError('MDX compile', err, prevEntry)
+				prevEntry.mdxComponent = null
+			}
 		}
 		return true
 	}

package/src/logger.js

@@ -18,10 +18,22 @@
  * under the License.
  */
 
+const resolveSupportColor = (stream) => {
+	if (typeof process === 'undefined') return false
+	const env = process.env || {}
+	if (env.NO_COLOR != null) return false
+	const force = env.FORCE_COLOR
+	if (force != null) {
+		return force !== '0'
+	}
+	if (stream && stream.isTTY) return true
+	const term = env.TERM || ''
+	if (!term || term.toLowerCase() === 'dumb') return false
+	return true
+}
+
 const supportColor =
-	typeof process !== 'undefined' &&
-	process.stdout &&
-	(process.stdout.isTTY || process.env.FORCE_COLOR)
+	resolveSupportColor(process.stdout) || resolveSupportColor(process.stderr)
 
 const formatter = (open, close, replace = open) =>
 	supportColor

package/src/mdx.js

@@ -131,9 +131,7 @@ const resolvePageHeadAssets = (page) => {
 export const buildPageContext = ({ routePath, path, pageMeta, pagesContext, lazyPagesTree = false }) => {
 	const page = pageMeta
 	const language = pagesContext.getLanguageForRoute ? pagesContext.getLanguageForRoute(routePath) : null
-	const getSiblings = pagesContext.getSiblings
-		? () => pagesContext.getSiblings(routePath, page.path || path)
-		: null
+	const getSiblings = pagesContext.getSiblings ? () => pagesContext.getSiblings(routePath, page.path || path) : null
 	if (page && getSiblings && page.getSiblings !== getSiblings) {
 		page.getSiblings = getSiblings
 	}
@@ -215,7 +213,7 @@ const normalizeStarryNightConfig = (value) => {
 	}
 	if (typeof value !== 'object') return null
 	const { enabled, options, ...rest } = value
-	if (enabled === false) return { enabled: false, options: null }
+	if (enabled === false) return { enabled: false, options }
 	if (options && typeof options === 'object') {
 		return { enabled: true, options: { ...options } }
 	}
@@ -289,12 +287,17 @@ const resolveMdxConfigForPage = async (frontmatter) => {
 	return mdxConfig
 }
 
-export const compileMdx = async ({ content, path, ctx }) => {
-	const mdxConfig = await resolveMdxConfigForPage(ctx.page.frontmatter)
-	const runtimeFactory = mdxConfig.development ? JSXDevFactory : JSXFactory
+export const compileMdxSource = async ({ content, path, frontmatter }) => {
+	const mdxConfig = await resolveMdxConfigForPage(frontmatter)
 	const compiled = await compile({ value: content, path: path }, mdxConfig)
+	const code = String(compiled.value ?? compiled)
+	return { code, development: Boolean(mdxConfig.development) }
+}
 
-	return await run(compiled, {
+export const runMdxSource = async ({ code, path, ctx, development = null }) => {
+	const isDev = development == null ? state.CURRENT_MODE !== 'production' : development
+	const runtimeFactory = isDev ? JSXDevFactory : JSXFactory
+	return await run(code, {
 		...runtimeFactory,
 		baseUrl: pathToFileURL(path).href,
 		ctx,
@@ -302,9 +305,23 @@ export const compileMdx = async ({ content, path, ctx }) => {
 	})
 }
 
+export const compileMdx = async ({ content, path, ctx }) => {
+	const result = await compileMdxSource({
+		content,
+		path,
+		frontmatter: ctx?.page?.frontmatter || null
+	})
+	return await runMdxSource({
+		code: result.code,
+		path,
+		ctx,
+		development: result.development
+	})
+}
+
 export const compilePageMdx = async (page, pagesContext, options = {}) => {
 	if (!page || page.content == null || page.mdxComponent) return
-	const { ctx = null, lazyPagesTree = false, refreshPagesTree = true } = options || {}
+	const { ctx = null, lazyPagesTree = false, refreshPagesTree = true, compiled = null } = options || {}
 	const activeCtx =
 		ctx ||
 		buildPageContext({
@@ -314,11 +331,18 @@ export const compilePageMdx = async (page, pagesContext, options = {}) => {
 			pagesContext,
 			lazyPagesTree
 		})
-	const mdxModule = await compileMdx({
-		content: page.content,
-		path: page.path,
-		ctx: activeCtx
-	})
+	const mdxModule = compiled?.code
+		? await runMdxSource({
+				code: compiled.code,
+				path: page.path,
+				ctx: activeCtx,
+				development: compiled.development
+			})
+		: await compileMdx({
+				content: page.content,
+				path: page.path,
+				ctx: activeCtx
+			})
 	page.mdxComponent = mdxModule.default
 	page.toc = mdxModule.toc
 	const shouldUseTocTitle = page.frontmatter?.title == null
@@ -346,41 +370,46 @@ export const renderHtml = async ({ routePath, path, components, pagesContext, pa
 		pageMeta,
 		pagesContext
 	})
-	await compilePageMdx(pageMeta, pagesContext, { ctx })
 
-	const [Head, Outlet] = createPortal()
-	const ExtraHead = () => {
-		return [
-			resolveRewindInject(),
-			...resolveUserHeadAssets(),
-			...resolvePageHeadAssets(pageMeta),
-			Outlet(),
-			RWND_FALLBACK
-		]
-	}
+	await compilePageMdx(pageMeta, pagesContext, { ctx })
 
+	const template = state.USER_THEME.template
 	const mdxComponent = pageMeta.mdxComponent
 
-	const PageContent = ({ components: extraComponents, ...props }, ...children) =>
-		mdxComponent({
-			children,
-			...props,
-			components: {
-				...components,
-				...extraComponents,
-				head: Head,
-				Head
-			}
-		})
+	const renderResult = await new Promise((resolve, reject) => {
+		const [Head, Outlet] = createPortal()
+		const ExtraHead = () => {
+			return [
+				resolveRewindInject(),
+				...resolveUserHeadAssets(),
+				...resolvePageHeadAssets(pageMeta),
+				Outlet(),
+				RWND_FALLBACK
+			]
+		}
 
-	const template = state.USER_THEME.template
+		const PageContent = ({ components: extraComponents, ...props }, ...children) => {
+			try {
+				return mdxComponent({
+					children,
+					...props,
+					components: {
+						...components,
+						...extraComponents,
+						head: Head,
+						Head
+					}
+				})
+			} catch (e) {
+				reject(e)
+			}
+		}
 
-	const renderResult = await new Promise((r) => {
 		const result = HTMLRenderer.c(
 			Suspense,
 			{
 				onLoad() {
-					nextTick(() => r(result))
+					nextTick(() => resolve(result))
 				}
 			},
 			() =>