packaton 0.0.1

package/.editorconfig

@@ -0,0 +1,11 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = tab
+insert_final_newline = true
+
+[*.md]
+indent_style = space

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Eric Fortis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,50 @@
+# Packaton WIP
+
+Static Pages Bundler.
+
+## HTML Template
+Optionally, you can create an HTML template.
+For example, to handle the common header, navigation, and footer.
+
+## Assets and CSP
+The production bundler inlines the JavaScript and CSS. Also, it
+computes their corresponding CSP nonce and injects it as well.
+
+
+## Images and Videos (immutable naming)
+For long-term caching, [media-remaper.js](src/media-remaper.js) appends a SHA-1 hash
+to the filenames and takes care of rewriting their `src` in HTML (**only in HTML**).
+
+If you want to use media files in CSS, create a similar function to
+`remapMediaInHTML` but with a regex for replacing the `url(...)` content.
+
+## Production Build
+It crawls the dev server, and saves each route as static html page.
+It saves the pages without the `.html` extension for pretty URLs. 
+See [Pretty routes for static HTML](https://blog.uxtly.com/pretty-routes-for-static-html)
+
+
+
+## Minifiers
+
+```js
+import { minify } from 'terser'
+
+
+const terserOptions = {}
+
+Packaton({
+  minifyJS: async js => (await minify(js, terserOptions)).code
+})
+```
+
+To avoid minifying, you can pass `a=>a` 
+
+
+## Caveats
+- can't write inline scripts or css (all must be in an external file, packaton inlines them)
+- must have an index
+- Ignored Documents start with `_`, so you can't have routes that begin with _
+- Non-Documents and Files outside .media are not automatically copied over,
+  you need to specify them. 
+- src/media only files at the top level get hashed-named. But files within subdirs are not (by design).

package/TODO.md

@@ -0,0 +1,8 @@
+# TODO
+- ignore as func in config
+- config file
+- config.media dir
+- cli? dev, build?
+- remap media in css and js
+- watch newly added routes
+- todo test on linux (open in browser too)

package/index.d.ts

@@ -0,0 +1,21 @@
+export interface Config {
+	mode?: 'development' | 'production';
+	srcPath?: string
+	ignore?: RegExp
+	
+
+	// Dev
+  host?: string,
+  port?: number
+  onReady?: (address: string) => void
+  hotReload?: boolean // For UI dev purposes only
+	
+	// Production
+	outputPath?: string
+	minifyJS?: (js: string) => Promise<string>
+	minifyCSS?: (css: string) => Promise<string>
+	minifyHTML?: (html: string) => Promise<string>
+	sitemapDomain?: string
+	cspMapEnabled?: boolean
+}
+  

package/index.js

@@ -0,0 +1,6 @@
+export { Packaton } from './src/app.js'
+export { HtmlCompiler } from './src/HtmlCompiler.js'
+export { minifyHTML } from './src/minifyHTML.js'
+export { minifyCSS } from './src/minifyCSS.js'
+export { minifyJS } from './src/minifyJS.js'
+export { reportSizes } from './src/reportSizes.js'

package/package.json

@@ -0,0 +1,23 @@
+{
+	"name": "packaton",
+	"version": "0.0.1",
+	"type": "module",
+	"author": "Eric Fortis",
+	"license": "MIT",
+	"scripts": {
+		"test": "node --test",
+		"outdated": "npm outdated --parseable | awk -F: '{ printf \"npm i %-30s ;# %s\\n\", $4, $2 }'"
+	},
+	"exports": {
+		".": {
+			"import": "./index.js",
+			"types": "./index.d.ts"
+		}
+	},
+	"peerDependencies": {
+		"terser": "^5"
+	},
+	"devDependencies": {
+		"terser": "5.44.1"
+	}
+}

package/src/HtmlCompiler.js

@@ -0,0 +1,132 @@
+import { join } from 'node:path'
+import { createHash } from 'node:crypto'
+
+import { read } from './fs-utils.js'
+import { remapMediaInHTML } from './media-remaper.js'
+
+
+export class HtmlCompiler {
+	html = ''
+	pSource = ''
+	css = ''
+	scriptsJs = ''
+	scriptsNonJs = ''
+	externalScripts = []
+	externalCSS = []
+	#minifyJS = a => a
+	#minifyCSS = a => a
+	#minifyHTML = a => a
+
+	constructor(html, pSource = '', { minifyJS, minifyCSS, minifyHTML }) {
+		this.html = html
+		this.pSource = pSource
+		this.#minifyJS = minifyJS
+		this.#minifyCSS = minifyCSS
+		this.#minifyHTML = minifyHTML
+	}
+
+	// Removes comments and format multi-line tags (needed for `removeLineContaining`)
+	async minifyHTML() {
+		this.html = await this.#minifyHTML(this.html)
+	}
+
+	remapMedia(mediaHashes) {
+		this.html = remapMediaInHTML(mediaHashes, this.html)
+	}
+
+	async inlineMinifiedCSS() {
+		for (const sheet of this.extractStyleSheetHrefs()) {
+			if (sheet.startsWith('http')) // TODO clean
+				this.externalCSS.push(sheet)
+			else {
+				this.css += read(join(this.pSource, sheet))
+				this.removeLineContaining(`href="${sheet}"`)
+			}
+		}
+		if (this.css) {
+			this.css = await this.#minifyCSS(this.css)
+			this.html = this.html.replace('<head>', `<head><style>${this.css}</style>`)
+		}
+	}
+
+	async inlineMinifiedJS() {
+		const scripts = []
+		for (const [src, type] of this.extractScriptSources()) {
+			if (src.startsWith('http')) // TODO clean
+				this.externalScripts.push(src)
+			else {
+				this.removeLineContaining(`src="${src}"`)
+				scripts.push([type, read(join(this.pSource, src))])
+			}
+		}
+
+		this.scriptsJs = await this.#minifyJS(scripts
+			.filter(([type]) => type === 'application/javascript')
+			.map(([, body]) => body)
+			.join('\n'))
+
+		this.scriptsNonJs = scripts
+			.filter(([type]) => type !== 'application/javascript')
+
+		if (this.scriptsJs)
+			this.html = this.html.replace('</body>', `<script>${this.scriptsJs}</script></body>`)
+		
+		for (const [type, body] of this.scriptsNonJs)
+			this.html = this.html.replace('</body>', `\n<script type="${type}">${body}</script></body>`)
+	}
+
+	csp() {
+		const cssHash = this.css
+			? `'${this.hash256(this.css)}'`
+			: '' // TODO maybe self?
+		const jsScriptHash = this.scriptsJs
+			? `'${this.hash256(this.scriptsJs)}'`
+			: '' // TODO maybe self?
+		const nonJsScriptHashes = this.scriptsNonJs
+			.map(([, body]) => `'${this.hash256(body)}'`).join(' ')
+		const externalScriptHashes = this.externalScripts.map(url => `${new URL(url).origin}`).join(' ')
+		return [
+			`default-src 'self'`,
+			`img-src 'self' data:`, // data: is for Safari's video player icons and for CSS bg images
+			`style-src ${cssHash}`,
+			`script-src ${nonJsScriptHashes} ${jsScriptHash} ${externalScriptHashes}`,
+			`frame-ancestors 'none'`
+		].join('; ')
+	}
+
+	hash256(data) {
+		return data
+			? 'sha256-' + createHash('sha256').update(data).digest('base64')
+			: ''
+	}
+
+	removeLineContaining(str) {
+		this.html = this.html.replace(new RegExp('^.*' + str + '.*\n', 'm'), '')
+	}
+
+	extractStyleSheetHrefs() {
+		const reExtractStyleSheets = /(?<=<link\s.*href=")[^"]+\.css/g
+		return Array.from(this.html.matchAll(reExtractStyleSheets), m => m[0])
+	}
+
+
+	/**
+	 * NOTE: Run minifyHTML first, because using any of these functions
+	 * because they don't support tags within comments nor multiline ones.
+	 * @returns {[src:string, type:string][]}
+	 */
+	extractScriptSources() {
+		const reExtractScripts = /<script\b([^>]*?)src="([^"]+)"([^>]*)>/g
+		return Array.from(this.html.matchAll(reExtractScripts), m => {
+			const pre = m[1]
+			const post = m[3]
+			const typeMatch = (pre + post).match(/type="([^"]+)"/)
+			return [
+				m[2], // src
+				typeMatch
+					? typeMatch[1]
+					: 'application/javascript'
+			]
+		})
+	}
+}

package/src/HtmlCompiler.test.js

@@ -0,0 +1,71 @@
+import { deepEqual } from 'node:assert/strict'
+import { describe, test } from 'node:test'
+import { HtmlCompiler } from './HtmlCompiler.js'
+import { minifyJS } from './minifyJS.js'
+import { minifyCSS } from './minifyCSS.js'
+import { minifyHTML } from './minifyHTML.js'
+
+const opts = {
+	minifyJS,
+	minifyCSS,
+	minifyHTML
+}
+
+describe('HtmlCompiler', () => {
+	const HTML = `
+<html lang="en">
+<head>
+	<link rel="stylesheet" href="0.css">
+	<link href="../1.css"  rel="stylesheet" >
+	<link href="2.css" rel="stylesheet">
+	<link rel="dns-prefetch" href="https://my.uxtly.com">
+</head>
+<body>
+
+<footer> Footer </footer>
+
+<script src="0.js"></script>
+	<script   src="../1.js"  ></script>
+	<script type="module"  src="2.js"></script>
+	<script src="3.json" type="speculationrules"></script>
+</body>
+</html>
+`
+
+	test('Extracts CSS files', () =>
+		deepEqual(new HtmlCompiler(HTML, '', opts).extractStyleSheetHrefs(), [
+			'0.css', '../1.css', '2.css'
+		]))
+
+	test('Extracts Script files', () =>
+		deepEqual(new HtmlCompiler(HTML, '', opts).extractScriptSources(), [
+			['0.js', 'application/javascript'],
+			['../1.js', 'application/javascript'],
+			['2.js', 'module'],
+			['3.json', 'speculationrules']
+		]))
+
+
+	test('Removes line containing X', () => {
+		const doc = new HtmlCompiler(HTML, '', opts)
+		doc.removeLineContaining('href="0.css"')
+		deepEqual(doc.html, `
+<html lang="en">
+<head>
+	<link href="../1.css"  rel="stylesheet" >
+	<link href="2.css" rel="stylesheet">
+	<link rel="dns-prefetch" href="https://my.uxtly.com">
+</head>
+<body>
+
+<footer> Footer </footer>
+
+<script src="0.js"></script>
+	<script   src="../1.js"  ></script>
+	<script type="module"  src="2.js"></script>
+	<script src="3.json" type="speculationrules"></script>
+</body>
+</html>
+`)
+	})
+})

package/src/WatcherDevClient.js

@@ -0,0 +1,18 @@
+import { join } from 'node:path'
+import { watch } from 'node:fs'
+import { EventEmitter } from 'node:events'
+import { docs } from './app.js'
+
+
+export const devClientWatcher = new class extends EventEmitter {
+	emit(file) { super.emit('RELOAD', file) }
+	subscribe(listener) { this.once('RELOAD', listener) }
+	unsubscribe(listener) { this.removeListener('RELOAD', listener) }
+}
+
+export function watchDev(rootPath) {
+	watch(rootPath, { recursive: true }, (_, file) => {
+		docs.onWatch(join(rootPath, file))
+		devClientWatcher.emit(file)
+	})
+}

package/src/app-dev.js

@@ -0,0 +1,26 @@
+import http from 'node:http'
+
+import { router } from './app-router.js'
+import { watchDev } from './WatcherDevClient.js'
+
+
+/**
+ * @param {Partial<Config>} config
+ * @returns {Promise<Server | undefined>}
+ */
+export function devStaticPages(config) {
+	return new Promise((resolve, reject) => {
+		if (config.hotReload)
+			watchDev(config.srcPath)
+
+		const server = http.createServer(router(config))
+		server.on('error', reject)
+		server.listen(config.port, config.host, () => {
+			const addr = `http://${server.address().address}:${server.address().port}`
+			config.onReady(addr)
+			console.log(addr)
+			resolve(server)
+		})
+	})
+}
+

package/src/app-prod.js

@@ -0,0 +1,100 @@
+import http from 'node:http'
+import { cpSync } from 'node:fs'
+import { basename, join, dirname } from 'node:path'
+
+import { docs } from './app.js'
+import { router } from './app-router.js'
+import { reportSizes } from './reportSizes.js'
+import { HtmlCompiler } from './HtmlCompiler.js'
+import { write, removeDir } from './fs-utils.js'
+import { renameMediaWithHashes } from './media-remaper.js'
+
+
+/**
+ * @param {Partial<Config>} opts
+ * @returns {Promise<unknown>}
+ */
+export async function buildStaticPages(config) {
+	return new Promise((resolve, reject) => {
+		const pSource = config.srcPath
+		const pDist = config.outputPath
+		const pDistMedia = join(config.outputPath, 'media')
+		const pDistSitemap = join(pDist, 'sitemap.txt')
+		const pDistCspNginxMap = join(pDist, '.csp-map.nginx')
+		const pSizesReport = 'packed-sizes.json'
+
+		const server = http.createServer(router(config))
+		server.listen(0, '127.0.0.1', async error => {
+			docs.init(config.srcPath, config.ignore)
+			try {
+				if (error) {
+					reject(error)
+					return
+				}
+
+				removeDir(pDist)
+				cpSync(join(pSource, 'media'), pDistMedia, {
+					recursive: true,
+					dereference: true,
+					filter(src) {
+						const f = basename(src)
+						return f !== '.DS_Store' && !f.startsWith('_')
+						// TODO 
+					}
+				})
+
+				const mediaHashes = await renameMediaWithHashes(pDistMedia) // only on top dir
+				const pages = await crawlRoutes(server.address(), docs.routes)
+
+				const cspByRoute = []
+				for (const [route, rawHtml] of pages) {
+					const doc = new HtmlCompiler(rawHtml, join(pSource, dirname(route)), config)
+					await doc.minifyHTML()
+					doc.remapMedia(mediaHashes)
+					// TODO remap media in css and js
+					await doc.inlineMinifiedCSS()
+					await doc.inlineMinifiedJS()
+					write(pDist + route, doc.html)
+					cspByRoute.push([route, doc.csp()])
+				}
+
+				if (config.sitemapDomain)
+					write(pDistSitemap, docs.routes
+						.filter(r => r !== '/index')
+						.map(r => `https://${config.sitemapDomain + r}`)
+						.join('\n'))
+
+				if (config.cspMapEnabled)
+					write(pDistCspNginxMap, cspByRoute.map(([route, csp]) =>
+						`${route} "${csp}";`).join('\n'))
+
+				reportSizes(pSizesReport, pDist, docs.routes)
+			}
+			catch (error) {
+				reject(error)
+				console.error(error)
+				process.exitCode = 1
+			}
+			finally {
+				server.close()
+				resolve()
+			}
+		})
+	})
+}
+
+
+async function crawlRoutes({ address, port }, routes) {
+	const pages = []
+	for (const route of routes)
+		try {
+			const resp = await fetch(`http://${address}:${port}` + route)
+			if (!resp.ok) throw resp.statusText
+			pages.push([route, await resp.text()])
+		}
+		catch (error) {
+			pages.push([route, ''])
+			console.warn(`Route: ${route} ${error?.message || error}`)
+		}
+	return pages
+}

package/src/app-router.js

@@ -0,0 +1,78 @@
+import { join } from 'node:path'
+import { readFile } from 'node:fs/promises'
+
+import { docs } from './app.js'
+import { mimeFor } from './mimes.js'
+import { devClientWatcher } from './WatcherDevClient.js'
+import { sendError, sendJSON, servePartialContent, serveStaticAsset } from './http-response.js'
+
+
+const API = {
+	watchDev: '/packaton/watch-dev'
+}
+
+const WATCHER_DEV = '/watcherDev.js'
+
+
+/** @param {Config} config */
+export function router({ srcPath, ignore, mode }) {
+	docs.init(srcPath, ignore)
+	return async function (req, response) {
+		let url = new URL(req.url, 'http://_').pathname
+		try {
+			if (url === API.watchDev) {
+				longPollDevHotReload(req, response)
+				return
+			}
+			if (url === WATCHER_DEV) {
+				serveStaticAsset(response, join(import.meta.dirname, url))
+				return
+			}
+
+			if (url === '/')
+				url = '/index'
+
+			const file = join(srcPath, url)
+			if (docs.hasRoute(url))
+				await serveDocument(response, docs.fileFor(url), mode === 'development')
+			else if (req.headers.range)
+				await servePartialContent(response, req.headers, file)
+			else
+				serveStaticAsset(response, file)
+		}
+		catch (error) {
+			sendError(response, error)
+		}
+	}
+}
+
+async function serveDocument(response, file, isDev) {
+	let html = file.endsWith('.html')
+		? await readFile(file, 'utf8')
+		: (await import(file + '?' + Date.now())).default()
+	if (isDev)
+		html += `<script type="module" src="${WATCHER_DEV}"></script>` 
+	response.setHeader('Content-Type', mimeFor('html'))
+	response.end(html)
+}
+
+
+const LONG_POLL_SERVER_TIMEOUT = 8000
+
+function longPollDevHotReload(req, response) {
+	function onDevChange(file) {
+		devClientWatcher.unsubscribe(onDevChange)
+		sendJSON(response, file)
+	}
+	response.setTimeout(LONG_POLL_SERVER_TIMEOUT, () => {
+		devClientWatcher.unsubscribe(onDevChange)
+		sendJSON(response, '')
+	})
+	req.on('error', () => {
+		devClientWatcher.unsubscribe(onDevChange)
+		response.destroy()
+	})
+	devClientWatcher.subscribe(onDevChange)
+}
+
+

package/src/app.js

@@ -0,0 +1,66 @@
+import { readdirSync } from 'node:fs'
+import { basename, join, dirname } from 'node:path'
+
+import { setup } from './config.js'
+import { isFile } from './fs-utils.js'
+import { devStaticPages } from './app-dev.js'
+import { buildStaticPages } from './app-prod.js'
+
+
+/**
+ * @param {Partial<Config>} opts
+ * @returns {Promise<Server | undefined>}
+ */
+export function Packaton(opts) {
+	const config = setup(opts)
+	return config.mode === 'development'
+		? devStaticPages(config)
+		: buildStaticPages(config)
+}
+
+
+export const docs = new class {
+	#srcPath = ''
+	#ignore = /.*/
+	#extensions = ['.html', '.html.js', '.html.ts']
+	#delay = Number(process.env.PACKATON_WATCHER_DEBOUNCE_MS ?? 80)
+
+	#routeToFileMap = new Map()
+	init(srcPath, ignore) {
+		this.#srcPath = srcPath
+		this.#ignore = ignore
+		this.#registerDocs()
+	}
+
+	get routes() { return Array.from(this.#routeToFileMap.keys()) }
+	fileFor = url => this.#routeToFileMap.get(url)
+	hasRoute = url => this.#routeToFileMap.has(url)
+
+	onWatch = /** @type {function} */ this.#debounce(f => {
+		if (this.#hasDocExt(f) && !this.hasRoute(this.#routeFor(f)) && isFile(f))
+			this.#registerDocs()
+	})
+
+	#registerDocs() {
+		const files = readdirSync(this.#srcPath, { recursive: true })
+			.filter(f => this.#hasDocExt(f) && !this.#ignore.test(f))
+
+		this.#routeToFileMap = new Map(files.map(f => [
+			this.#routeFor(f),
+			join(this.#srcPath, f)
+		]))
+	}
+
+	#routeFor = f => '/' + join(dirname(f), this.#removeDocExt(f))
+	#hasDocExt = f => this.#extensions.some(ext => f.endsWith(ext))
+	#findDocExt = f => this.#extensions.find(ext => f.endsWith(ext))
+	#removeDocExt = f => basename(f, this.#findDocExt(f))
+
+	#debounce(fn) {
+		let timer
+		return arg => {
+			clearTimeout(timer)
+			timer = setTimeout(() => fn(arg), this.#delay)
+		}
+	}
+}