@sveltejs/kit 1.0.0-next.43 → 1.0.0-next.430

package/src/core/prerender/crawl.js

@@ -0,0 +1,194 @@
+const DOCTYPE = 'DOCTYPE';
+const CDATA_OPEN = '[CDATA[';
+const CDATA_CLOSE = ']]>';
+const COMMENT_OPEN = '--';
+const COMMENT_CLOSE = '-->';
+
+const TAG_OPEN = /[a-zA-Z]/;
+const TAG_CHAR = /[a-zA-Z0-9]/;
+const ATTRIBUTE_NAME = /[^\t\n\f />"'=]/;
+
+const WHITESPACE = /[\s\n\r]/;
+
+/** @param {string} html */
+export function crawl(html) {
+	/** @type {string[]} */
+	const hrefs = [];
+
+	let i = 0;
+	main: while (i < html.length) {
+		const char = html[i];
+
+		if (char === '<') {
+			if (html[i + 1] === '!') {
+				i += 2;
+
+				if (html.slice(i, i + DOCTYPE.length).toUpperCase() === DOCTYPE) {
+					i += DOCTYPE.length;
+					while (i < html.length) {
+						if (html[i++] === '>') {
+							continue main;
+						}
+					}
+				}
+
+				// skip cdata
+				if (html.slice(i, i + CDATA_OPEN.length) === CDATA_OPEN) {
+					i += CDATA_OPEN.length;
+					while (i < html.length) {
+						if (html.slice(i, i + CDATA_CLOSE.length) === CDATA_CLOSE) {
+							i += CDATA_CLOSE.length;
+							continue main;
+						}
+
+						i += 1;
+					}
+				}
+
+				// skip comments
+				if (html.slice(i, i + COMMENT_OPEN.length) === COMMENT_OPEN) {
+					i += COMMENT_OPEN.length;
+					while (i < html.length) {
+						if (html.slice(i, i + COMMENT_CLOSE.length) === COMMENT_CLOSE) {
+							i += COMMENT_CLOSE.length;
+							continue main;
+						}
+
+						i += 1;
+					}
+				}
+			}
+
+			// parse opening tags
+			const start = ++i;
+			if (TAG_OPEN.test(html[start])) {
+				while (i < html.length) {
+					if (!TAG_CHAR.test(html[i])) {
+						break;
+					}
+
+					i += 1;
+				}
+
+				const tag = html.slice(start, i).toUpperCase();
+
+				if (tag === 'SCRIPT' || tag === 'STYLE') {
+					while (i < html.length) {
+						if (
+							html[i] === '<' &&
+							html[i + 1] === '/' &&
+							html.slice(i + 2, i + 2 + tag.length).toUpperCase() === tag
+						) {
+							continue main;
+						}
+
+						i += 1;
+					}
+				}
+
+				let href = '';
+				let rel = '';
+
+				while (i < html.length) {
+					const start = i;
+
+					const char = html[start];
+					if (char === '>') break;
+
+					if (ATTRIBUTE_NAME.test(char)) {
+						i += 1;
+
+						while (i < html.length) {
+							if (!ATTRIBUTE_NAME.test(html[i])) {
+								break;
+							}
+
+							i += 1;
+						}
+
+						const name = html.slice(start, i).toLowerCase();
+
+						while (WHITESPACE.test(html[i])) i += 1;
+
+						if (html[i] === '=') {
+							i += 1;
+							while (WHITESPACE.test(html[i])) i += 1;
+
+							let value;
+
+							if (html[i] === "'" || html[i] === '"') {
+								const quote = html[i++];
+
+								const start = i;
+								let escaped = false;
+
+								while (i < html.length) {
+									if (!escaped) {
+										const char = html[i];
+
+										if (html[i] === quote) {
+											break;
+										}
+
+										if (char === '\\') {
+											escaped = true;
+										}
+									}
+
+									i += 1;
+								}
+
+								value = html.slice(start, i);
+							} else {
+								const start = i;
+								while (html[i] !== '>' && !WHITESPACE.test(html[i])) i += 1;
+								value = html.slice(start, i);
+
+								i -= 1;
+							}
+
+							if (name === 'href') {
+								href = value;
+							} else if (name === 'rel') {
+								rel = value;
+							} else if (name === 'src') {
+								hrefs.push(value);
+							} else if (name === 'srcset') {
+								const candidates = [];
+								let insideURL = true;
+								value = value.trim();
+								for (let i = 0; i < value.length; i++) {
+									if (value[i] === ',' && (!insideURL || (insideURL && value[i + 1] === ' '))) {
+										candidates.push(value.slice(0, i));
+										value = value.substring(i + 1).trim();
+										i = 0;
+										insideURL = true;
+									} else if (value[i] === ' ') {
+										insideURL = false;
+									}
+								}
+								candidates.push(value);
+								for (const candidate of candidates) {
+									const src = candidate.split(WHITESPACE)[0];
+									hrefs.push(src);
+								}
+							}
+						} else {
+							i -= 1;
+						}
+					}
+
+					i += 1;
+				}
+
+				if (href && !/\bexternal\b/i.test(rel)) {
+					hrefs.push(href);
+				}
+			}
+		}
+
+		i += 1;
+	}
+
+	return hrefs;
+}

package/src/core/prerender/prerender.js

@@ -0,0 +1,378 @@
+import { readFileSync, writeFileSync } from 'fs';
+import { dirname, join } from 'path';
+import { pathToFileURL, URL } from 'url';
+import { mkdirp, posixify, walk } from '../../utils/filesystem.js';
+import { installPolyfills } from '../../node/polyfills.js';
+import { is_root_relative, resolve } from '../../utils/url.js';
+import { queue } from './queue.js';
+import { crawl } from './crawl.js';
+import { escape_html_attr } from '../../utils/escape.js';
+import { logger } from '../utils.js';
+import { load_config } from '../config/index.js';
+
+/**
+ * @typedef {import('types').PrerenderErrorHandler} PrerenderErrorHandler
+ * @typedef {import('types').Logger} Logger
+ */
+
+const [, , client_out_dir, results_path, manifest_path, verbose] = process.argv;
+
+prerender();
+
+/**
+ * @param {Parameters<PrerenderErrorHandler>[0]} details
+ * @param {import('types').ValidatedKitConfig} config
+ */
+function format_error({ status, path, referrer, referenceType }, config) {
+	const message =
+		status === 404 && !path.startsWith(config.paths.base)
+			? `${path} does not begin with \`base\`, which is configured in \`paths.base\` and can be imported from \`$app/paths\``
+			: path;
+
+	return `${status} ${message}${referrer ? ` (${referenceType} from ${referrer})` : ''}`;
+}
+
+/**
+ * @param {Logger} log
+ * @param {import('types').ValidatedKitConfig} config
+ * @returns {PrerenderErrorHandler}
+ */
+function normalise_error_handler(log, config) {
+	switch (config.prerender.onError) {
+		case 'continue':
+			return (details) => {
+				log.error(format_error(details, config));
+			};
+		case 'fail':
+			return (details) => {
+				throw new Error(format_error(details, config));
+			};
+		default:
+			return config.prerender.onError;
+	}
+}
+
+const OK = 2;
+const REDIRECT = 3;
+
+/**
+ * @param {import('types').Prerendered} prerendered
+ */
+const output_and_exit = (prerendered) => {
+	writeFileSync(
+		results_path,
+		JSON.stringify(prerendered, (_key, value) =>
+			value instanceof Map ? Array.from(value.entries()) : value
+		)
+	);
+	process.exit(0);
+};
+
+export async function prerender() {
+	/** @type {import('types').Prerendered} */
+	const prerendered = {
+		pages: new Map(),
+		assets: new Map(),
+		redirects: new Map(),
+		paths: []
+	};
+
+	/** @type {import('types').ValidatedKitConfig} */
+	const config = (await load_config()).kit;
+
+	if (!config.prerender.enabled) {
+		output_and_exit(prerendered);
+		return;
+	}
+
+	/** @type {import('types').Logger} */
+	const log = logger({
+		verbose: verbose === 'true'
+	});
+
+	installPolyfills();
+	const { fetch } = globalThis;
+	globalThis.fetch = async (info, init) => {
+		/** @type {string} */
+		let url;
+
+		/** @type {RequestInit} */
+		let opts = {};
+
+		if (info instanceof Request) {
+			url = info.url;
+
+			opts = {
+				method: info.method,
+				headers: info.headers,
+				body: info.body,
+				mode: info.mode,
+				credentials: info.credentials,
+				cache: info.cache,
+				redirect: info.redirect,
+				referrer: info.referrer,
+				integrity: info.integrity
+			};
+		} else {
+			url = info.toString();
+		}
+
+		if (url.startsWith(config.prerender.origin + '/')) {
+			const request = new Request(url, opts);
+			const response = await server.respond(request, {
+				getClientAddress,
+				prerendering: {
+					dependencies: new Map()
+				}
+			});
+
+			const decoded = new URL(url).pathname;
+
+			save(
+				'dependencies',
+				response,
+				Buffer.from(await response.clone().arrayBuffer()),
+				decoded,
+				encodeURI(decoded),
+				null,
+				'fetched'
+			);
+
+			return response;
+		}
+
+		return fetch(info, init);
+	};
+
+	const server_root = join(config.outDir, 'output');
+
+	/** @type {import('types').ServerModule} */
+	const { Server, override } = await import(pathToFileURL(`${server_root}/server/index.js`).href);
+
+	/** @type {import('types').SSRManifest} */
+	const manifest = (await import(pathToFileURL(`${server_root}/server/manifest.js`).href)).manifest;
+
+	override({
+		paths: config.paths,
+		prerendering: true,
+		read: (file) => readFileSync(join(config.files.assets, file))
+	});
+
+	const server = new Server(manifest);
+
+	const error = normalise_error_handler(log, config);
+
+	const q = queue(config.prerender.concurrency);
+
+	/**
+	 * @param {string} path
+	 * @param {boolean} is_html
+	 */
+	function output_filename(path, is_html) {
+		const file = path.slice(config.paths.base.length + 1) || 'index.html';
+
+		if (is_html && !file.endsWith('.html')) {
+			return file + (file.endsWith('/') ? 'index.html' : '.html');
+		}
+
+		return file;
+	}
+
+	const files = new Set(walk(client_out_dir).map(posixify));
+	const seen = new Set();
+	const written = new Set();
+
+	/**
+	 * @param {string | null} referrer
+	 * @param {string} decoded
+	 * @param {string} [encoded]
+	 */
+	function enqueue(referrer, decoded, encoded) {
+		if (seen.has(decoded)) return;
+		seen.add(decoded);
+
+		const file = decoded.slice(config.paths.base.length + 1);
+		if (files.has(file)) return;
+
+		return q.add(() => visit(decoded, encoded || encodeURI(decoded), referrer));
+	}
+
+	/**
+	 * @param {string} decoded
+	 * @param {string} encoded
+	 * @param {string?} referrer
+	 */
+	async function visit(decoded, encoded, referrer) {
+		if (!decoded.startsWith(config.paths.base)) {
+			error({ status: 404, path: decoded, referrer, referenceType: 'linked' });
+			return;
+		}
+
+		/** @type {Map<string, import('types').PrerenderDependency>} */
+		const dependencies = new Map();
+
+		const response = await server.respond(new Request(config.prerender.origin + encoded), {
+			getClientAddress,
+			prerendering: {
+				dependencies
+			}
+		});
+
+		const body = Buffer.from(await response.arrayBuffer());
+
+		save('pages', response, body, decoded, encoded, referrer, 'linked');
+
+		for (const [dependency_path, result] of dependencies) {
+			// this seems circuitous, but using new URL allows us to not care
+			// whether dependency_path is encoded or not
+			const encoded_dependency_path = new URL(dependency_path, 'http://localhost').pathname;
+			const decoded_dependency_path = decodeURI(encoded_dependency_path);
+
+			const body = result.body ?? new Uint8Array(await result.response.arrayBuffer());
+			save(
+				'dependencies',
+				result.response,
+				body,
+				decoded_dependency_path,
+				encoded_dependency_path,
+				decoded,
+				'fetched'
+			);
+		}
+
+		if (config.prerender.crawl && response.headers.get('content-type') === 'text/html') {
+			for (const href of crawl(body.toString())) {
+				if (href.startsWith('data:') || href.startsWith('#')) continue;
+
+				const resolved = resolve(encoded, href);
+				if (!is_root_relative(resolved)) continue;
+
+				const { pathname, search } = new URL(resolved, 'http://localhost');
+
+				if (search) {
+					// TODO warn that query strings have no effect on statically-exported pages
+				}
+
+				enqueue(decoded, decodeURI(pathname), pathname);
+			}
+		}
+	}
+
+	/**
+	 * @param {'pages' | 'dependencies'} category
+	 * @param {Response} response
+	 * @param {string | Uint8Array} body
+	 * @param {string} decoded
+	 * @param {string} encoded
+	 * @param {string | null} referrer
+	 * @param {'linked' | 'fetched'} referenceType
+	 */
+	function save(category, response, body, decoded, encoded, referrer, referenceType) {
+		const response_type = Math.floor(response.status / 100);
+		const type = /** @type {string} */ (response.headers.get('content-type'));
+		const is_html = response_type === REDIRECT || type === 'text/html';
+
+		const file = output_filename(decoded, is_html);
+		const dest = `${config.outDir}/output/prerendered/${category}/${file}`;
+
+		if (written.has(file)) return;
+
+		if (response_type === REDIRECT) {
+			const location = response.headers.get('location');
+
+			if (location) {
+				const resolved = resolve(encoded, location);
+				if (is_root_relative(resolved)) {
+					enqueue(decoded, decodeURI(resolved), resolved);
+				}
+
+				if (!response.headers.get('x-sveltekit-normalize')) {
+					mkdirp(dirname(dest));
+
+					log.warn(`${response.status} ${decoded} -> ${location}`);
+
+					writeFileSync(
+						dest,
+						`<meta http-equiv="refresh" content=${escape_html_attr(`0;url=${location}`)}>`
+					);
+
+					written.add(file);
+
+					if (!prerendered.redirects.has(decoded)) {
+						prerendered.redirects.set(decoded, {
+							status: response.status,
+							location: resolved
+						});
+
+						prerendered.paths.push(decoded);
+					}
+				}
+			} else {
+				log.warn(`location header missing on redirect received from ${decoded}`);
+			}
+
+			return;
+		}
+
+		if (response.status === 200) {
+			mkdirp(dirname(dest));
+
+			log.info(`${response.status} ${decoded}`);
+			writeFileSync(dest, body);
+			written.add(file);
+
+			if (is_html) {
+				prerendered.pages.set(decoded, {
+					file
+				});
+			} else {
+				prerendered.assets.set(decoded, {
+					type
+				});
+			}
+
+			prerendered.paths.push(decoded);
+		} else if (response_type !== OK) {
+			error({ status: response.status, path: decoded, referrer, referenceType });
+		}
+	}
+
+	if (config.prerender.enabled) {
+		for (const entry of config.prerender.entries) {
+			if (entry === '*') {
+				/** @type {import('types').ManifestData} */
+				const { routes } = (await import(pathToFileURL(manifest_path).href)).manifest._;
+				const entries = routes
+					.map((route) => (route.type === 'page' && !route.id.includes('[') ? `/${route.id}` : ''))
+					.filter(Boolean);
+
+				for (const entry of entries) {
+					enqueue(null, config.paths.base + entry); // TODO can we pre-normalize these?
+				}
+			} else {
+				enqueue(null, config.paths.base + entry);
+			}
+		}
+
+		await q.done();
+	}
+
+	const rendered = await server.respond(new Request(config.prerender.origin + '/[fallback]'), {
+		getClientAddress,
+		prerendering: {
+			fallback: true,
+			dependencies: new Map()
+		}
+	});
+
+	const file = `${config.outDir}/output/prerendered/fallback.html`;
+	mkdirp(dirname(file));
+	writeFileSync(file, await rendered.text());
+
+	output_and_exit(prerendered);
+}
+
+/** @return {string} */
+function getClientAddress() {
+	throw new Error('Cannot read clientAddress during prerendering');
+}

package/src/core/prerender/queue.js

@@ -0,0 +1,80 @@
+/**
+ * @typedef {{
+ *   fn: () => Promise<any>,
+ *   fulfil: (value: any) => void,
+ *   reject: (error: Error) => void
+ * }} Task
+ */
+
+/** @param {number} concurrency */
+export function queue(concurrency) {
+	/** @type {Task[]} */
+	const tasks = [];
+
+	let current = 0;
+
+	/** @type {(value?: any) => void} */
+	let fulfil;
+
+	/** @type {(error: Error) => void} */
+	let reject;
+
+	let closed = false;
+
+	const done = new Promise((f, r) => {
+		fulfil = f;
+		reject = r;
+	});
+
+	done.catch(() => {
+		// this is necessary in case a catch handler is never added
+		// to the done promise by the user
+	});
+
+	function dequeue() {
+		if (current < concurrency) {
+			const task = tasks.shift();
+
+			if (task) {
+				current += 1;
+				const promise = Promise.resolve(task.fn());
+
+				promise
+					.then(task.fulfil, (err) => {
+						task.reject(err);
+						reject(err);
+					})
+					.then(() => {
+						current -= 1;
+						dequeue();
+					});
+			} else if (current === 0) {
+				closed = true;
+				fulfil();
+			}
+		}
+	}
+
+	return {
+		/** @param {() => any} fn */
+		add: (fn) => {
+			if (closed) throw new Error('Cannot add tasks to a queue that has ended');
+
+			const promise = new Promise((fulfil, reject) => {
+				tasks.push({ fn, fulfil, reject });
+			});
+
+			dequeue();
+			return promise;
+		},
+
+		done: () => {
+			if (current === 0) {
+				closed = true;
+				fulfil();
+			}
+
+			return done;
+		}
+	};
+}