methanol 0.0.21 → 0.0.22

package/bin/methanol.js

@@ -20,5 +20,4 @@
  * under the License.
  */
 
-import '../src/register-loader.js'
-import('../src/main.js')
+import('../src/entry.js')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "methanol",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "description": "Static site generator powered by rEFui and MDX",
   "main": "./index.js",
   "type": "module",
@@ -29,21 +29,23 @@
     "@mdx-js/mdx": "^3.1.1",
     "@sindresorhus/fnv1a": "^3.1.0",
     "@stefanprobst/rehype-extract-toc": "^3.0.0",
-    "@wooorm/starry-night": "^3.8.0",
+    "@wooorm/starry-night": "^3.9.0",
     "chokidar": "^5.0.0",
     "esbuild": "^0.27.2",
+    "fast-glob": "^3.3.3",
     "gray-matter": "^4.0.3",
     "hast-util-is-element": "^3.0.0",
+    "htmlparser2": "^10.1.0",
     "json5": "^2.2.3",
     "null-prototype-object": "^1.2.5",
+    "picomatch": "^4.0.3",
     "refui": "^0.17.1",
     "refurbish": "^0.1.8",
     "rehype-slug": "^6.0.0",
     "rehype-starry-night": "^2.2.0",
     "remark-gfm": "^4.0.1",
-    "unist-util-visit": "^5.0.0",
+    "unist-util-visit": "^5.1.0",
     "vite": "^7.3.1",
-    "vite-plugin-pwa": "^1.2.0",
     "workbox-core": "^7.4.0",
     "workbox-routing": "^7.4.0",
     "workbox-strategies": "^7.4.0",

package/src/base.js

@@ -0,0 +1,36 @@
+/* Copyright Yukino Song, SudoMaker Ltd.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * 	http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const normalizeBasePrefix = (value) => {
+	if (!value || value === '/' || value === './') return ''
+	if (typeof value !== 'string') return ''
+	let base = value.trim()
+	if (!base || base === '/' || base === './') return ''
+	if (base.startsWith('http://') || base.startsWith('https://')) {
+		try {
+			base = new URL(base).pathname
+		} catch {
+			return ''
+		}
+	}
+	if (!base.startsWith('/')) return ''
+	if (base.endsWith('/')) base = base.slice(0, -1)
+	return base
+}

package/src/build-system.js

@@ -18,20 +18,22 @@
  * under the License.
  */
 
+import { existsSync } from 'fs'
 import { writeFile, mkdir, rm, readFile, readdir, stat } from 'fs/promises'
-import { resolve, dirname, join } from 'path'
+import { resolve, dirname, join, basename } from 'path'
+import { createHash } from 'crypto'
 import { fileURLToPath } from 'url'
 import { build as viteBuild, mergeConfig, normalizePath } from 'vite'
-import { VitePWA } from 'vite-plugin-pwa'
 import { state, cli } from './state.js'
 import { resolveUserViteConfig } from './config.js'
 import { buildPagesContext } from './pages.js'
 import { selectFeedPages } from './feed.js'
 import { buildComponentRegistry } from './components.js'
 import { createBuildWorkers, runWorkerStage, terminateWorkers } from './workers/build-pool.js'
-import { methanolVirtualHtmlPlugin, methanolResolverPlugin } from './vite-plugins.js'
+import { methanolResolverPlugin } from './vite-plugins.js'
 import { createStageLogger } from './stage-logger.js'
 import { preparePublicAssets } from './public-assets.js'
+export { scanHtmlEntries, rewriteHtmlEntries } from './html/build-html.js'
 
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = dirname(__filename)
@@ -40,11 +42,29 @@ const ensureDir = async (dir) => {
 	await mkdir(dir, { recursive: true })
 }
 
+const ensureSwEntry = async () => {
+	const entriesDir = resolve(resolveMethanolDir(), ENTRY_DIR)
+	await ensureDir(entriesDir)
+	const swEntryPath = resolve(entriesDir, 'sw-entry.js')
+	const swSource = normalizePath(resolve(__dirname, 'client', 'sw.js'))
+	await writeFile(swEntryPath, `import ${JSON.stringify(swSource)}\n`)
+	return swEntryPath
+}
+
+const INLINE_DIR = 'inline'
+const ENTRY_DIR = 'entries'
+const WRITE_CONCURRENCY_LIMIT = 32
+
+const resolveMethanolDir = () => resolve(state.PAGES_DIR, '.methanol')
+
 const isHtmlFile = (name) => name.endsWith('.html')
 const collectHtmlFiles = async (dir, basePath = '') => {
 	const entries = await readdir(dir)
 	const files = []
 	for (const entry of entries.sort()) {
+		if (entry.startsWith('.') || entry.startsWith('_')) {
+			continue
+		}
 		const fullPath = resolve(dir, entry)
 		const stats = await stat(fullPath)
 		if (stats.isDirectory()) {
@@ -65,11 +85,18 @@ const collectHtmlFiles = async (dir, basePath = '') => {
 	return files
 }
 
-export const buildHtmlEntries = async () => {
+const hashKey = (value) =>
+	createHash('md5').update(value).digest('hex')
+
+const makeInputKey = (prefix, value) => `${prefix}-${hashKey(value).slice(0, 12)}`
+
+export const buildHtmlEntries = async (options = {}) => {
+	const keepWorkers = Boolean(options.keepWorkers)
 	await resolveUserViteConfig('build') // Prepare `base`
-	if (state.INTERMEDIATE_DIR) {
-		await rm(state.INTERMEDIATE_DIR, { recursive: true, force: true })
-		await ensureDir(state.INTERMEDIATE_DIR)
+	const htmlStageDir = state.INTERMEDIATE_DIR || resolve(state.PAGES_DIR, '.methanol/html')
+	if (htmlStageDir) {
+		await rm(htmlStageDir, { recursive: true, force: true })
+		await ensureDir(htmlStageDir)
 	}
 
 	const logEnabled = state.CURRENT_MODE === 'production' && cli.command === 'build' && !cli.CLI_VERBOSE
@@ -84,8 +111,13 @@ export const buildHtmlEntries = async () => {
 	}
 	await buildComponentRegistry()
 	const pagesContext = await buildPagesContext({ compileAll: false })
-	const entry = {}
-	const htmlCache = new Map()
+	const htmlEntries = []
+	const htmlEntryNames = new Set()
+	const inlineDir = resolve(resolveMethanolDir(), INLINE_DIR)
+	await rm(inlineDir, { recursive: true, force: true })
+	await ensureDir(inlineDir)
+	const renderScans = new Map()
+	const renderScansById = new Map()
 	const resolveOutputName = (page) => {
 		if (page.routePath === '/') return 'index'
 		if (page.isIndex && page.dir) {
@@ -94,15 +126,16 @@ export const buildHtmlEntries = async () => {
 		return page.routePath.slice(1)
 	}
 
-	const pages = pagesContext.pages || []
+	const pages = pagesContext.pagesAll || pagesContext.pages || []
 	const totalPages = pages.length
 	const { workers, assignments } = createBuildWorkers(totalPages)
+	const writeConcurrency = Math.max(1, Math.floor(WRITE_CONCURRENCY_LIMIT / workers.length))
 	const excludedRoutes = Array.from(pagesContext.excludedRoutes || [])
 	const excludedDirs = Array.from(pagesContext.excludedDirs || [])
 	const rssContent = new Map()
-	const intermediateOutputs = []
 	let feedIds = []
 	let feedAssignments = null
+	let completedRun = false
 	try {
 		await runWorkerStage({
 			workers,
@@ -193,7 +226,9 @@ export const buildHtmlEntries = async () => {
 					type: 'render',
 					stage: 'render',
 					ids: assignments[index],
-					feedIds: feedAssignments ? feedAssignments[index] : []
+					feedIds: feedAssignments ? feedAssignments[index] : [],
+					htmlStageDir,
+					writeConcurrency
 				}
 			})),
 			onProgress: (count) => {
@@ -205,14 +240,15 @@ export const buildHtmlEntries = async () => {
 				if (!result || typeof result.id !== 'number') return
 				const page = pages[result.id]
 				if (!page) return
-				const html = result.html
-				const name = resolveOutputName(page)
-				const id = normalizePath(resolve(state.VIRTUAL_HTML_OUTPUT_ROOT, `${name}.html`))
-				entry[name] = id
-				htmlCache.set(id, html)
-				if (state.INTERMEDIATE_DIR) {
-					intermediateOutputs.push({ name, id })
+				if (result.scan) {
+					renderScansById.set(result.id, result.scan)
 				}
+				const name = resolveOutputName(page)
+				const outPath = htmlStageDir
+					? (result.stagePath || resolve(htmlStageDir, `${name}.html`))
+					: `${name}.html`
+				htmlEntryNames.add(name)
+				htmlEntries.push({ name, routePath: page.routePath, stagePath: outPath, source: 'rendered' })
 				if (result.feedContent != null) {
 					const key = page.path || page.routePath
 					if (key) {
@@ -222,16 +258,18 @@ export const buildHtmlEntries = async () => {
 			}
 		})
 		stageLogger.end(renderToken)
+
+		for (const [id, scan] of renderScansById.entries()) {
+			const page = pages[id]
+			if (!page || !scan) continue
+			const name = resolveOutputName(page)
+			const stagePath = htmlStageDir ? resolve(htmlStageDir, `${name}.html`) : `${name}.html`
+			renderScans.set(stagePath, scan)
+		}
+		completedRun = true
 	} finally {
-		await terminateWorkers(workers)
-	}
-	if (state.INTERMEDIATE_DIR) {
-		for (const output of intermediateOutputs) {
-			const html = htmlCache.get(output.id)
-			if (typeof html !== 'string') continue
-			const outPath = resolve(state.INTERMEDIATE_DIR, `${output.name}.html`)
-			await ensureDir(dirname(outPath))
-			await writeFile(outPath, html)
+		if (!keepWorkers || !completedRun) {
+			await terminateWorkers(workers)
 		}
 	}
 
@@ -255,27 +293,133 @@ export const buildHtmlEntries = async () => {
 		}
 		const name = file.relativePath.replace(/\.html$/, '')
 		const outputName = name === 'index' ? 'index' : name
-		if (entry[outputName]) {
+		if (htmlEntryNames.has(outputName)) {
 			continue
 		}
 		const html = await readFile(file.fullPath, 'utf-8')
-		const id = normalizePath(resolve(state.VIRTUAL_HTML_OUTPUT_ROOT, `${outputName}.html`))
-		entry[outputName] = id
-		htmlCache.set(id, html)
-		if (state.INTERMEDIATE_DIR) {
-			const outPath = resolve(state.INTERMEDIATE_DIR, file.relativePath)
+		const outPath = htmlStageDir ? resolve(htmlStageDir, file.relativePath) : null
+		if (outPath) {
 			await ensureDir(dirname(outPath))
 			await writeFile(outPath, html)
 		}
+		htmlEntryNames.add(outputName)
+		htmlEntries.push({
+			name: outputName,
+			routePath: outputName === 'index'
+				? '/'
+				: outputName.endsWith('/index')
+					? `/${outputName.slice(0, -'/index'.length)}/`
+					: `/${outputName}`,
+			stagePath: outPath,
+			inputPath: file.fullPath,
+			source: 'static'
+		})
 	}
 
-	return { entry, htmlCache, pagesContext, rssContent }
+	return {
+		htmlEntries,
+		htmlStageDir,
+		pagesContext,
+		rssContent,
+		renderScans,
+		renderScansById,
+		workers: keepWorkers ? workers : null,
+		assignments: keepWorkers ? assignments : null
+	}
+}
+
+export const rewriteHtmlEntriesInWorkers = async ({
+	pages = [],
+	htmlStageDir,
+	manifest,
+	scanResult,
+	renderScansById,
+	onProgress,
+	workers: existingWorkers = null,
+	assignments: existingAssignments = null
+}) => {
+	const totalPages = pages.length
+	if (!totalPages) return
+	const useExisting = Array.isArray(existingWorkers) && Array.isArray(existingAssignments)
+	const { workers, assignments } = useExisting
+		? { workers: existingWorkers, assignments: existingAssignments }
+		: createBuildWorkers(totalPages)
+	try {
+		if (!useExisting) {
+			await runWorkerStage({
+				workers,
+				stage: 'setPagesLite',
+				messages: workers.map((worker) => ({
+					worker,
+					message: {
+						type: 'setPagesLite',
+						stage: 'setPagesLite',
+						pages
+					}
+				}))
+			})
+		}
+
+		const entryModules = Array.isArray(scanResult?.entryModules) ? scanResult.entryModules : []
+		const commonScripts = Array.isArray(scanResult?.commonScripts) ? scanResult.commonScripts : []
+		const commonEntry = scanResult?.commonScriptEntry?.manifestKey
+			? manifest?.[scanResult.commonScriptEntry.manifestKey] || manifest?.[`/${scanResult.commonScriptEntry.manifestKey}`]
+			: null
+
+		await runWorkerStage({
+			workers,
+			stage: 'rewrite',
+			messages: workers.map((worker, index) => {
+				const ids = assignments[index] || []
+				const scans = {}
+				if (renderScansById) {
+					for (const id of ids) {
+						const scan = renderScansById.get(id)
+						if (scan) scans[id] = scan
+					}
+				}
+				return {
+					worker,
+					message: {
+						type: 'rewrite',
+						stage: 'rewrite',
+						ids,
+						htmlStageDir,
+						manifest,
+						entryModules,
+						commonScripts,
+						commonEntry,
+						scans
+					}
+				}
+			}),
+			onProgress: (count) => {
+				if (typeof onProgress === 'function') {
+					onProgress(count, totalPages)
+				}
+			}
+		})
+	} finally {
+		if (!useExisting) {
+			await terminateWorkers(workers)
+		}
+	}
 }
 
-export const runViteBuild = async (entry, htmlCache) => {
+export const runViteBuild = async (inputs) => {
 	const logEnabled = state.CURRENT_MODE === 'production' && cli.command === 'build' && !cli.CLI_VERBOSE
 	const stageLogger = createStageLogger(logEnabled)
 	const token = stageLogger.start('Building bundle')
+	const rewriteOptions = inputs.rewrite || null
+	const preWrite = typeof inputs.preWrite === 'function' ? inputs.preWrite : null
+	const postWrite = typeof inputs.postWrite === 'function' ? inputs.postWrite : null
+	let manifestData = null
+	let bundleEnded = false
+	const endBundleStage = () => {
+		if (bundleEnded) return
+		bundleEnded = true
+		stageLogger.end(token)
+	}
 
 	if (state.STATIC_DIR !== false && state.MERGED_ASSETS_DIR) {
 		await preparePublicAssets({
@@ -285,20 +429,53 @@ export const runViteBuild = async (entry, htmlCache) => {
 		})
 	}
 	const copyPublicDirEnabled = state.STATIC_DIR !== false
-	const manifestConfig = state.PWA_OPTIONS?.manifest || {}
-	const resolvedManifest = { name: state.SITE_NAME, short_name: state.SITE_NAME, ...manifestConfig }
+	const entryModules = Array.isArray(inputs.entryModules) ? inputs.entryModules : []
+	const entryInputs = entryModules
+		.filter((entry) => entry && entry.kind !== 'style')
+		.map((entry) => entry.fsPath)
+		.filter(Boolean)
+		.sort()
+	const htmlEntries = Array.isArray(inputs.htmlEntries) ? inputs.htmlEntries : []
+	const htmlInputs = htmlEntries
+		.filter((entry) => entry?.source === 'static' && entry.inputPath)
+		.map((entry) => entry.inputPath)
+		.sort()
+	if (cli.CLI_VERBOSE && entryInputs.length === 0) {
+		console.log('Vite pipeline: no wrapper entries detected (no module scripts/stylesheets found)')
+	}
+	const rollupInput = {}
+	for (const entryPath of entryInputs) {
+		const normalized = normalizePath(entryPath)
+		rollupInput[makeInputKey('chunk', normalized)] = normalized
+	}
+	for (const htmlPath of htmlInputs) {
+		const normalized = normalizePath(htmlPath)
+		rollupInput[makeInputKey('html', normalized)] = normalized
+	}
+	let swEntryPath = null
+	if (state.PWA_ENABLED) {
+		swEntryPath = await ensureSwEntry()
+		if (swEntryPath) {
+			const normalized = normalizePath(swEntryPath)
+			rollupInput['sw'] = normalized
+		}
+	}
 	const baseConfig = {
 		configFile: false,
 		root: state.PAGES_DIR,
-		appType: 'mpa',
+		appType: 'custom',
 		publicDir: state.STATIC_DIR === false ? false : state.STATIC_DIR,
 		logLevel: cli.CLI_VERBOSE ? 'info' : 'silent',
 		build: {
 			outDir: state.DIST_DIR,
 			emptyOutDir: true,
 			rollupOptions: {
-				input: entry
+				input: rollupInput,
+				output: {
+					entryFileNames: (chunk) => (chunk.name === 'sw' ? 'sw.js' : 'assets/[name]-[hash].js')
+				}
 			},
+			manifest: true,
 			copyPublicDir: copyPublicDirEnabled,
 			minify: true
 		},
@@ -309,42 +486,115 @@ export const runViteBuild = async (entry, htmlCache) => {
 		resolve: {
 			dedupe: ['refui', 'methanol']
 		},
-		plugins: [
-			methanolVirtualHtmlPlugin(htmlCache),
-			methanolResolverPlugin(),
-			state.PWA_ENABLED
-				? VitePWA({
-						injectRegister: 'auto',
-						registerType: 'autoUpdate',
-						strategies: 'injectManifest',
-						srcDir: resolve(__dirname, 'client'),
-						filename: 'sw.js',
-						manifest: resolvedManifest,
-						injectManifest: {
-							globPatterns: ['**/*.{js,css,html,ico,png,svg}'],
-							...(state.PWA_OPTIONS?.injectManifest || {})
-						}
-					})
-				: null
-		]
+		plugins: [methanolResolverPlugin()]
 	}
 	const userConfig = await resolveUserViteConfig('build')
 	const finalConfig = userConfig ? mergeConfig(baseConfig, userConfig) : baseConfig
 
-	const originalLog = console.log
-	const originalWarn = console.warn
-	if (!cli.CLI_VERBOSE) {
-		console.log = () => {}
-		console.warn = () => {}
+	// Keep the pipeline deterministic: do not let user configs override the build root/output/inputs.
+	finalConfig.root = state.PAGES_DIR
+	finalConfig.appType = 'custom'
+	finalConfig.publicDir = state.STATIC_DIR === false ? false : state.STATIC_DIR
+	finalConfig.build = {
+		...(finalConfig.build || {}),
+		outDir: state.DIST_DIR,
+		emptyOutDir: true,
+		manifest: finalConfig.build?.manifest === undefined ? true : finalConfig.build.manifest,
+		copyPublicDir: copyPublicDirEnabled,
+		rollupOptions: {
+			...((finalConfig.build && finalConfig.build.rollupOptions) || {}),
+			input: rollupInput,
+			output: (() => {
+				const existing = finalConfig.build?.rollupOptions?.output
+				const outputConfig = Array.isArray(existing) ? existing[0] || {} : existing || {}
+				return {
+					...outputConfig,
+					entryFileNames: (chunk) => (chunk.name === 'sw' ? 'sw.js' : 'assets/[name]-[hash].js')
+				}
+			})()
+		}
+	}
+
+	const manifestFileName = typeof finalConfig.build?.manifest === 'string'
+		? finalConfig.build.manifest
+		: '.vite/manifest.json'
+	const manifestPath = resolve(state.DIST_DIR, manifestFileName.replace(/^\//, ''))
+	const manifestEnabled = finalConfig.build?.manifest !== false
+	let rewriteDone = false
+	let preWriteDone = false
+	let postWriteDone = false
+	const loadManifest = async () => {
+		if (!manifestEnabled) return null
+		if (!existsSync(manifestPath)) return null
+		const raw = await readFile(manifestPath, 'utf-8')
+		return JSON.parse(raw)
+	}
+	const runPreWrite = async () => {
+		if (rewriteOptions) {
+			endBundleStage()
+		}
+		if (preWrite) {
+			await preWrite({ manifest: manifestData })
+		}
+	}
+	const runPostWrite = async () => {
+		if (postWrite) {
+			await postWrite({ manifest: manifestData })
+		}
+	}
+	const runRewrite = async () => {
+		if (!rewriteOptions || rewriteDone || !manifestData) return
+		const rewriteToken = logEnabled ? stageLogger.start('Rewriting HTML') : null
+		try {
+			await rewriteHtmlEntriesInWorkers({
+				...rewriteOptions,
+				manifest: manifestData,
+				onProgress: (done, total) => {
+					if (!rewriteToken) return
+					stageLogger.update(rewriteToken, `Rewriting HTML [${done}/${total}]`)
+				}
+			})
+		} finally {
+			rewriteDone = true
+			if (rewriteToken) {
+				stageLogger.end(rewriteToken)
+			}
+		}
+	}
+
+	const postBundlePlugin = {
+		name: 'methanol:post-bundle',
+		apply: 'build',
+		enforce: 'post',
+		async writeBundle() {
+			manifestData = await loadManifest()
+			await runPreWrite()
+			await runRewrite()
+			await runPostWrite()
+		}
 	}
 
+	finalConfig.plugins = [...(finalConfig.plugins || []), postBundlePlugin]
+
+	await viteBuild(finalConfig)
+	endBundleStage()
+	const methanolManifestDir = resolve(state.PAGES_DIR, '.methanol')
+	const methanolManifestPath = resolve(methanolManifestDir, 'manifest.json')
 	try {
-		await viteBuild(finalConfig)
-	} finally {
-		if (!cli.CLI_VERBOSE) {
-			console.log = originalLog
-			console.warn = originalWarn
+		const parsed = manifestData || (manifestEnabled ? JSON.parse(await readFile(manifestPath, 'utf-8')) : null)
+		if (!parsed) return {}
+		await ensureDir(methanolManifestDir)
+		await writeFile(methanolManifestPath, JSON.stringify(parsed, null, 2))
+		await rm(manifestPath, { force: true })
+		const manifestDir = dirname(manifestPath)
+		if (basename(manifestDir) === '.vite') {
+			await rm(manifestDir, { recursive: true, force: true })
+		}
+		return parsed
+	} catch (error) {
+		if (cli.CLI_VERBOSE) {
+			console.log(`Vite pipeline: failed to read manifest at ${manifestPath}`)
 		}
+		return {}
 	}
-	stageLogger.end(token)
 }