methanol 0.0.10 → 0.0.12

package/src/pages.js

@@ -22,9 +22,11 @@ import matter from 'gray-matter'
 import { readdir, readFile, stat } from 'fs/promises'
 import { existsSync } from 'fs'
 import { resolve, join, relative } from 'path'
+import { cpus } from 'os'
+import { Worker } from 'worker_threads'
 import { state, cli } from './state.js'
 import { withBase } from './config.js'
-import { compilePageMdx } from './mdx.js'
+import { compileMdxSource, compilePageMdx } from './mdx.js'
 import { createStageLogger } from './stage-logger.js'
 
 const isPageFile = (name) => name.endsWith('.mdx') || name.endsWith('.md')
@@ -32,6 +34,137 @@ const isIgnoredEntry = (name) => name.startsWith('.') || name.startsWith('_')
 
 const pageMetadataCache = new Map()
 const pageDerivedCache = new Map()
+const MDX_WORKER_URL = new URL('./workers/mdx-compile-worker.js', import.meta.url)
+const cliOverrides = {
+	CLI_INTERMEDIATE_DIR: cli.CLI_INTERMEDIATE_DIR,
+	CLI_EMIT_INTERMEDIATE: cli.CLI_EMIT_INTERMEDIATE,
+	CLI_HOST: cli.CLI_HOST,
+	CLI_PORT: cli.CLI_PORT,
+	CLI_PAGES_DIR: cli.CLI_PAGES_DIR,
+	CLI_COMPONENTS_DIR: cli.CLI_COMPONENTS_DIR,
+	CLI_ASSETS_DIR: cli.CLI_ASSETS_DIR,
+	CLI_OUTPUT_DIR: cli.CLI_OUTPUT_DIR,
+	CLI_CONFIG_PATH: cli.CLI_CONFIG_PATH,
+	CLI_SITE_NAME: cli.CLI_SITE_NAME,
+	CLI_CODE_HIGHLIGHTING: cli.CLI_CODE_HIGHLIGHTING,
+	CLI_JOBS: cli.CLI_JOBS,
+	CLI_VERBOSE: cli.CLI_VERBOSE,
+	CLI_BASE: cli.CLI_BASE,
+	CLI_SEARCH: cli.CLI_SEARCH,
+	CLI_PWA: cli.CLI_PWA
+}
+
+const resolveWorkerCount = (total) => {
+	const cpuCount = Math.max(1, cpus()?.length || 1)
+	const requested = state.WORKER_JOBS
+	if (requested == null || requested <= 0) {
+		const items = Math.max(1, Number.isFinite(total) ? total : 1)
+		const autoCount = Math.round(Math.log(items))
+		return Math.max(1, Math.min(cpuCount, autoCount))
+	}
+	return Math.max(1, Math.min(cpuCount, Math.floor(requested)))
+}
+
+const compileMdxSources = async (pages, options = {}) => {
+	const targets = pages.filter((page) => page && page.content != null && !page.mdxComponent)
+	const results = new Map()
+	if (!targets.length) return results
+	const { onProgress } = options || {}
+	const reportProgress = (page) => {
+		if (typeof onProgress === 'function') {
+			onProgress(page)
+		}
+	}
+	const workerCount = Math.min(resolveWorkerCount(targets.length), targets.length)
+	if (workerCount <= 1) {
+		for (const page of targets) {
+			const result = await compileMdxSource({
+				content: page.content,
+				path: page.path,
+				frontmatter: page.frontmatter
+			})
+			results.set(page, result)
+			reportProgress(page)
+		}
+		return results
+	}
+
+	return await new Promise((resolve, reject) => {
+		const workers = []
+		const pending = new Map()
+		let cursor = 0
+		let nextId = 0
+		let finished = false
+
+		const finalize = async (error) => {
+			if (finished) return
+			finished = true
+			await Promise.all(workers.map((worker) => worker.terminate().catch(() => null)))
+			if (error) {
+				reject(error)
+				return
+			}
+			resolve(results)
+		}
+
+		const assign = (worker) => {
+			if (cursor >= targets.length) return false
+			const page = targets[cursor++]
+			const id = nextId++
+			pending.set(id, page)
+			worker.postMessage({
+				id,
+				path: page.path,
+				content: page.content,
+				frontmatter: page.frontmatter
+			})
+			return true
+		}
+
+		const handleMessage = (worker, message) => {
+			if (finished) return
+			const { id, result, error } = message || {}
+			const page = pending.get(id)
+			pending.delete(id)
+			if (!page) return
+			if (error) {
+				void finalize(new Error(error))
+				return
+			}
+			results.set(page, result)
+			reportProgress(page)
+			assign(worker)
+			if (results.size === targets.length && pending.size === 0) {
+				void finalize()
+			}
+		}
+
+		const handleError = (error) => {
+			if (finished) return
+			void finalize(error instanceof Error ? error : new Error(String(error)))
+		}
+
+		for (let i = 0; i < workerCount; i += 1) {
+			const worker = new Worker(MDX_WORKER_URL, {
+				type: 'module',
+				workerData: {
+					mode: state.CURRENT_MODE,
+					configPath: cli.CLI_CONFIG_PATH,
+					cli: cliOverrides
+				}
+			})
+			workers.push(worker)
+			worker.on('message', (message) => handleMessage(worker, message))
+			worker.on('error', handleError)
+			worker.on('exit', (code) => {
+				if (code !== 0) {
+					handleError(new Error(`MDX worker exited with code ${code}`))
+				}
+			})
+			assign(worker)
+		}
+	})
+}
 
 const collectLanguagesFromPages = (pages = []) => {
 	const languages = new Map()
@@ -202,6 +335,10 @@ const buildPagesTree = (pages, options = {}) => {
 		if (!rootPrefix) return normalizeRoutePath(localPath)
 		return normalizeRoutePath(`${rootPrefix}${localPath}`)
 	}
+	const resolveDirFsPath = (dir) => {
+		if (!rootDir) return resolve(state.PAGES_DIR, dir)
+		return resolve(state.PAGES_DIR, join(rootDir, dir))
+	}
 	const currentRouteWithinRoot = resolveRouteWithinRoot(currentRoutePath)
 	const isUnderRoot = (page) => {
 		if (!rootDir) return true
@@ -270,7 +407,7 @@ const buildPagesTree = (pages, options = {}) => {
 		const dir = {
 			type: 'directory',
 			name,
-			path: resolve(state.PAGES_DIR, path),
+			path: resolveDirFsPath(path),
 			children: [],
 			depth,
 			routePath: buildDirRoutePath(path),
@@ -612,47 +749,12 @@ const buildNavSequence = (nodes, pagesByRoute) => {
 	return result
 }
 
-export const buildPagesContext = async ({ compileAll = true } = {}) => {
-	const logEnabled = state.CURRENT_MODE === 'production' && cli.command === 'build'
-	const stageLogger = createStageLogger(logEnabled)
-	const collectToken = stageLogger.start('Collecting pages')
-	const collected = await collectPages()
-	stageLogger.end(collectToken)
-	let pages = collected.pages
-	const excludedRoutes = collected.excludedRoutes
-	const excludedDirs = collected.excludedDirs
-	const hasIndex = pages.some((page) => page.routePath === '/')
-	if (!hasIndex) {
-		const content = buildIndexFallback(pages, state.SITE_NAME)
-		pages.unshift({
-			routePath: '/',
-			routeHref: withBase('/'),
-			path: resolve(state.PAGES_DIR, 'index.md'),
-			relativePath: 'index.md',
-			name: 'index',
-			dir: '',
-			segments: [],
-			depth: 0,
-			isIndex: true,
-			title: state.SITE_NAME || 'Methanol Site',
-			weight: null,
-			date: null,
-			isRoot: false,
-			hidden: false,
-			content,
-			frontmatter: {},
-			matter: null,
-			stats: { size: content.length, createdAt: null, updatedAt: null },
-			createdAt: null,
-			updatedAt: null
-		})
-		if (excludedRoutes?.has('/')) {
-			excludedRoutes.delete('/')
-		}
-	}
-
+export const createPagesContextFromPages = ({ pages, excludedRoutes, excludedDirs } = {}) => {
+	const pageList = Array.isArray(pages) ? pages : []
+	const routeExcludes = excludedRoutes || new Set()
+	const dirExcludes = excludedDirs || new Set()
 	const pagesByRoute = new Map()
-	for (const page of pages) {
+	for (const page of pageList) {
 		if (!pagesByRoute.has(page.routePath)) {
 			pagesByRoute.set(page.routePath, page)
 		}
@@ -699,7 +801,7 @@ export const buildPagesContext = async ({ compileAll = true } = {}) => {
 	}
 	let pagesTree = getPagesTree('/')
 	const notFound = pagesByRoute.get('/404') || null
-	const languages = collectLanguagesFromPages(pages)
+	const languages = collectLanguagesFromPages(pageList)
 	const userSite = state.USER_SITE || {}
 	const siteBase = state.VITE_BASE ?? userSite.base ?? null
 	const site = {
@@ -719,9 +821,9 @@ export const buildPagesContext = async ({ compileAll = true } = {}) => {
 		},
 		generatedAt: new Date().toISOString()
 	}
-	const excludedDirPaths = new Set(Array.from(excludedDirs).map((dir) => `/${dir}`))
+	const excludedDirPaths = new Set(Array.from(dirExcludes).map((dir) => `/${dir}`))
 	const pagesContext = {
-		pages,
+		pages: pageList,
 		pagesByRoute,
 		getPageByRoute,
 		pagesTree,
@@ -771,31 +873,96 @@ export const buildPagesContext = async ({ compileAll = true } = {}) => {
 			pagesContext.getLanguageForRoute = (routePath) =>
 				resolveLanguageForRoute(pagesContext.languages, routePath)
 		},
-		excludedRoutes,
-		excludedDirs,
+		excludedRoutes: routeExcludes,
+		excludedDirs: dirExcludes,
 		excludedDirPaths,
 		notFound,
 		languages,
 		getLanguageForRoute: (routePath) => resolveLanguageForRoute(languages, routePath),
 		site
 	}
+	return pagesContext
+}
+
+export const buildPagesContext = async ({ compileAll = true } = {}) => {
+	const logEnabled = state.CURRENT_MODE === 'production' && cli.command === 'build'
+	const stageLogger = createStageLogger(logEnabled)
+	const collectToken = stageLogger.start('Collecting pages')
+	const collected = await collectPages()
+	stageLogger.end(collectToken)
+	let pages = collected.pages
+	const excludedRoutes = collected.excludedRoutes
+	const excludedDirs = collected.excludedDirs
+	const hasIndex = pages.some((page) => page.routePath === '/')
+	if (!hasIndex) {
+		const content = buildIndexFallback(pages, state.SITE_NAME)
+		pages.unshift({
+			routePath: '/',
+			routeHref: withBase('/'),
+			path: resolve(state.PAGES_DIR, 'index.md'),
+			relativePath: 'index.md',
+			name: 'index',
+			dir: '',
+			segments: [],
+			depth: 0,
+			isIndex: true,
+			title: state.SITE_NAME || 'Methanol Site',
+			weight: null,
+			date: null,
+			isRoot: false,
+			hidden: false,
+			content,
+			frontmatter: {},
+			matter: null,
+			stats: { size: content.length, createdAt: null, updatedAt: null },
+			createdAt: null,
+			updatedAt: null
+		})
+		if (excludedRoutes?.has('/')) {
+			excludedRoutes.delete('/')
+		}
+	}
+
+	const pagesContext = createPagesContextFromPages({
+		pages,
+		excludedRoutes,
+		excludedDirs
+	})
 	if (compileAll) {
 		const compileToken = stageLogger.start('Compiling MDX')
-		const totalPages = pages.length
-		for (let i = 0; i < pages.length; i++) {
-			const page = pages[i]
-			if (logEnabled) {
-				stageLogger.update(compileToken, `Compiling MDX [${i + 1}/${totalPages}] ${page.path}`)
+		const compileTargets = pages.filter((page) => page && page.content != null && !page.mdxComponent)
+		const totalPages = compileTargets.length
+		let completed = 0
+		const compiledSources = await compileMdxSources(compileTargets, {
+			onProgress: (page) => {
+				if (!logEnabled) return
+				completed += 1
+				stageLogger.update(
+					compileToken,
+					`Compiling MDX [${completed}/${totalPages}] ${page.routePath || page.path}`
+				)
 			}
+		})
+		stageLogger.end(compileToken)
+		const executeToken = stageLogger.start('Running MDX')
+		completed = 0
+		for (const page of compileTargets) {
+			const compiled = compiledSources.get(page) || null
 			await compilePageMdx(page, pagesContext, {
 				lazyPagesTree: true,
-				refreshPagesTree: false
+				refreshPagesTree: false,
+				compiled
 			})
+			if (logEnabled) {
+				completed += 1
+				stageLogger.update(
+					executeToken,
+					`Running MDX [${completed}/${totalPages}] ${page.routePath || page.path}`
+				)
+			}
 		}
-		stageLogger.end(compileToken)
-		pagesTreeCache.clear()
-		pagesTree = getPagesTree('/')
-		pagesContext.pagesTree = pagesTree
+		stageLogger.end(executeToken)
+		pagesContext.refreshPagesTree?.()
 	}
 	return pagesContext
 }

package/src/state.js

@@ -104,14 +104,14 @@ const withCommonOptions = (y) =>
 		.option('highlight', {
 			describe: 'Enable or disable code highlighting',
 			type: 'boolean',
-			coerce: (value) => {
-				if (value == null) return null
-				if (typeof value === 'boolean') return value
-				const normalized = String(value).trim().toLowerCase()
-				if (normalized === 'true') return true
-				if (normalized === 'false') return false
-				return null
-			}
+			default: undefined
+		})
+		.option('jobs', {
+			alias: 'j',
+			describe: 'Worker thread count (0 for auto)',
+			type: 'number',
+			requiresArg: true,
+			nargs: 1
 		})
 		.option('verbose', {
 			alias: 'v',
@@ -126,12 +126,14 @@ const withCommonOptions = (y) =>
 			nargs: 1
 		})
 		.option('search', {
-			describe: 'Enable search indexing (pagefind)',
-			type: 'boolean'
+			describe: 'Enable or disable search indexing (pagefind)',
+			type: 'boolean',
+			default: undefined
 		})
 		.option('pwa', {
-			describe: 'Enable PWA support',
-			type: 'boolean'
+			describe: 'Enable or disable PWA support',
+			type: 'boolean',
+			default: undefined
 		})
 
 const parser = yargs(hideBin(process.argv))
@@ -161,10 +163,11 @@ export const cli = {
 	CLI_CONFIG_PATH: argv.config || null,
 	CLI_SITE_NAME: argv['site-name'] || null,
 	CLI_CODE_HIGHLIGHTING: typeof argv.highlight === 'boolean' ? argv.highlight : null,
+	CLI_JOBS: typeof argv.jobs === 'number' && Number.isFinite(argv.jobs) ? argv.jobs : null,
 	CLI_VERBOSE: Boolean(argv.verbose),
 	CLI_BASE: argv.base || null,
-	CLI_SEARCH: argv.search,
-	CLI_PWA: argv.pwa
+	CLI_SEARCH: typeof argv.search === 'boolean' ? argv.search : undefined,
+	CLI_PWA: typeof argv.pwa === 'boolean' ? argv.pwa : undefined
 }
 
 export const state = {
@@ -204,6 +207,7 @@ export const state = {
 	THEME_POST_BUNDLE_HOOKS: [],
 	STARRY_NIGHT_ENABLED: false,
 	STARRY_NIGHT_OPTIONS: null,
+	WORKER_JOBS: 0,
 	GFM_ENABLED: true,
 	PWA_ENABLED: false,
 	PWA_OPTIONS: null,

package/src/workers/build-pool.js

@@ -0,0 +1,156 @@
+/* Copyright Yukino Song, SudoMaker Ltd.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * 	http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { cpus } from 'os'
+import { Worker } from 'worker_threads'
+import { state, cli } from '../state.js'
+
+const BUILD_WORKER_URL = new URL('./build-worker.js', import.meta.url)
+const cliOverrides = {
+	CLI_INTERMEDIATE_DIR: cli.CLI_INTERMEDIATE_DIR,
+	CLI_EMIT_INTERMEDIATE: cli.CLI_EMIT_INTERMEDIATE,
+	CLI_HOST: cli.CLI_HOST,
+	CLI_PORT: cli.CLI_PORT,
+	CLI_PAGES_DIR: cli.CLI_PAGES_DIR,
+	CLI_COMPONENTS_DIR: cli.CLI_COMPONENTS_DIR,
+	CLI_ASSETS_DIR: cli.CLI_ASSETS_DIR,
+	CLI_OUTPUT_DIR: cli.CLI_OUTPUT_DIR,
+	CLI_CONFIG_PATH: cli.CLI_CONFIG_PATH,
+	CLI_SITE_NAME: cli.CLI_SITE_NAME,
+	CLI_CODE_HIGHLIGHTING: cli.CLI_CODE_HIGHLIGHTING,
+	CLI_JOBS: cli.CLI_JOBS,
+	CLI_VERBOSE: cli.CLI_VERBOSE,
+	CLI_BASE: cli.CLI_BASE,
+	CLI_SEARCH: cli.CLI_SEARCH,
+	CLI_PWA: cli.CLI_PWA
+}
+
+const resolveWorkerCount = (total) => {
+	const cpuCount = Math.max(1, cpus()?.length || 1)
+	const requested = state.WORKER_JOBS
+	if (requested == null || requested <= 0) {
+		const items = Math.max(1, Number.isFinite(total) ? total : 1)
+		const autoCount = Math.round(Math.log(items))
+		return Math.max(1, Math.min(cpuCount, autoCount))
+	}
+	return Math.max(1, Math.min(cpuCount, Math.floor(requested)))
+}
+
+export const createBuildWorkers = (pageCount, options = {}) => {
+	const { command = 'build' } = options || {}
+	const workerCount = Math.min(resolveWorkerCount(pageCount), pageCount || 1) || 1
+	const workers = []
+	for (let i = 0; i < workerCount; i += 1) {
+		workers.push(
+			new Worker(BUILD_WORKER_URL, {
+				type: 'module',
+				workerData: {
+					mode: state.CURRENT_MODE,
+					configPath: cli.CLI_CONFIG_PATH,
+					command,
+					cli: cliOverrides
+				}
+			})
+		)
+	}
+	const assignments = Array.from({ length: workers.length }, () => [])
+	for (let i = 0; i < pageCount; i += 1) {
+		assignments[i % workers.length].push(i)
+	}
+	return { workers, assignments }
+}
+
+export const terminateWorkers = async (workers = []) => {
+	await Promise.all(workers.map((worker) => worker.terminate().catch(() => null)))
+}
+
+export const runWorkerStage = async ({ workers, stage, messages, onProgress, collect }) => {
+	return await new Promise((resolve, reject) => {
+		let completed = 0
+		let doneCount = 0
+		const results = []
+		const handleFailure = (error) => {
+			for (const w of workers) {
+				const handler = handlers.get(w)
+				if (handler) {
+					w.off('message', handler)
+					w.off('error', errorHandlers.get(w))
+					w.off('exit', exitHandlers.get(w))
+				}
+			}
+			reject(error instanceof Error ? error : new Error(String(error)))
+		}
+		const handleMessage = (worker, message) => {
+			if (!message || message.stage !== stage) return
+			if (message.type === 'progress') {
+				completed += 1
+				if (onProgress) {
+					onProgress(completed)
+				}
+				return
+			}
+			if (message.type === 'done') {
+				if (collect) {
+					const data = collect(message)
+					if (Array.isArray(data) && data.length) {
+						results.push(...data)
+					}
+				}
+				doneCount += 1
+				if (doneCount >= workers.length) {
+					for (const w of workers) {
+						const handler = handlers.get(w)
+						if (handler) {
+							w.off('message', handler)
+							w.off('error', errorHandlers.get(w))
+							w.off('exit', exitHandlers.get(w))
+						}
+					}
+					resolve(results)
+				}
+				return
+			}
+			if (message.type === 'error') {
+				handleFailure(new Error(message.error || 'Worker error'))
+			}
+		}
+		const handlers = new Map()
+		const errorHandlers = new Map()
+		const exitHandlers = new Map()
+		for (const worker of workers) {
+			const handler = (message) => handleMessage(worker, message)
+			handlers.set(worker, handler)
+			worker.on('message', handler)
+			const errorHandler = (error) => handleFailure(error)
+			const exitHandler = (code) => {
+				if (code !== 0) {
+					handleFailure(new Error(`Build worker exited with code ${code}`))
+				}
+			}
+			errorHandlers.set(worker, errorHandler)
+			exitHandlers.set(worker, exitHandler)
+			worker.on('error', errorHandler)
+			worker.on('exit', exitHandler)
+		}
+		for (const entry of messages) {
+			entry.worker.postMessage(entry.message)
+		}
+	})
+}