@cloudflare/pages-shared 0.0.0-ship-js

package/metadata-generator/createMetadataObject.ts

@@ -0,0 +1,164 @@
+import {
+	ANALYTICS_VERSION,
+	REDIRECTS_VERSION,
+	HEADERS_VERSION,
+	SPLAT_REGEX,
+	PLACEHOLDER_REGEX,
+} from "./constants";
+import type { MetadataStaticRedirects } from "../asset-server/metadata";
+import type {
+	Metadata,
+	MetadataRedirects,
+	MetadataHeaders,
+	ParsedRedirects,
+	ParsedHeaders,
+	Logger,
+} from "./types";
+
+export function createMetadataObject({
+	redirects,
+	headers,
+	webAnalyticsToken,
+	deploymentId,
+	failOpen,
+	logger = (_message: string) => {},
+}: {
+	redirects?: ParsedRedirects;
+	headers?: ParsedHeaders;
+	webAnalyticsToken?: string;
+	deploymentId?: string;
+	failOpen?: boolean;
+	logger?: Logger;
+}): Metadata {
+	return {
+		...constructRedirects({ redirects, logger }),
+		...constructHeaders({ headers, logger }),
+		...constructWebAnalytics({ webAnalyticsToken, logger }),
+		deploymentId,
+		failOpen,
+	};
+}
+
+function constructRedirects({
+	redirects,
+	logger,
+}: {
+	redirects?: ParsedRedirects;
+	logger: Logger;
+}): Metadata {
+	if (!redirects) return {};
+
+	const num_valid = redirects.rules.length;
+	const num_invalid = redirects.invalid.length;
+
+	logger(
+		`Parsed ${num_valid} valid redirect rule${num_valid === 1 ? "" : "s"}.`
+	);
+
+	if (num_invalid > 0) {
+		logger(`Found invalid redirect lines:`);
+		for (const { line, lineNumber, message } of redirects.invalid) {
+			if (line) logger(`  - ${lineNumber ? `#${lineNumber}: ` : ""}${line}`);
+			logger(`    ${message}`);
+		}
+	}
+
+	/* Better to return no Redirects object at all than one with empty rules */
+	if (num_valid === 0) {
+		return {};
+	}
+
+	const staticRedirects: MetadataStaticRedirects = {};
+	const dynamicRedirects: MetadataRedirects = {};
+	let canCreateStaticRule = true;
+	for (const rule of redirects.rules) {
+		if (!rule.from.match(SPLAT_REGEX) && !rule.from.match(PLACEHOLDER_REGEX)) {
+			if (canCreateStaticRule) {
+				staticRedirects[rule.from] = {
+					status: rule.status,
+					to: rule.to,
+					lineNumber: rule.lineNumber,
+				};
+				continue;
+			} else {
+				logger(
+					`Info: the redirect rule ${rule.from} → ${rule.status} ${rule.to} could be made more performant by bringing it above any lines with splats or placeholders.`
+				);
+			}
+		}
+
+		dynamicRedirects[rule.from] = { status: rule.status, to: rule.to };
+		canCreateStaticRule = false;
+	}
+
+	return {
+		redirects: {
+			version: REDIRECTS_VERSION,
+			staticRules: staticRedirects,
+			rules: dynamicRedirects,
+		},
+	};
+}
+
+function constructHeaders({
+	headers,
+	logger,
+}: {
+	headers?: ParsedHeaders;
+	logger: Logger;
+}): Metadata {
+	if (!headers) return {};
+
+	const num_valid = headers.rules.length;
+	const num_invalid = headers.invalid.length;
+
+	logger(`Parsed ${num_valid} valid header rule${num_valid === 1 ? "" : "s"}.`);
+
+	if (num_invalid > 0) {
+		logger(`Found invalid header lines:`);
+		for (const { line, lineNumber, message } of headers.invalid) {
+			if (line) logger(`  - ${lineNumber ? `#${lineNumber}: ` : ""} ${line}`);
+			logger(`    ${message}`);
+		}
+	}
+
+	/* Better to return no Headers object at all than one with empty rules */
+	if (num_valid === 0) {
+		return {};
+	}
+
+	const rules: MetadataHeaders = {};
+	for (const rule of headers.rules) {
+		rules[rule.path] = {};
+
+		if (Object.keys(rule.headers).length) {
+			rules[rule.path].set = rule.headers;
+		}
+		if (rule.unsetHeaders.length) {
+			rules[rule.path].unset = rule.unsetHeaders;
+		}
+	}
+
+	return {
+		headers: {
+			version: HEADERS_VERSION,
+			rules,
+		},
+	};
+}
+
+function constructWebAnalytics({
+	webAnalyticsToken,
+}: {
+	webAnalyticsToken?: string;
+	logger: Logger;
+}) {
+	if (!webAnalyticsToken) return {};
+
+	return {
+		analytics: {
+			version: ANALYTICS_VERSION,
+			token: webAnalyticsToken,
+		},
+	};
+}

package/metadata-generator/parseHeaders.ts

@@ -0,0 +1,168 @@
+import {
+	MAX_LINE_LENGTH,
+	MAX_HEADER_RULES,
+	HEADER_SEPARATOR,
+	UNSET_OPERATOR,
+} from "./constants";
+import { validateUrl } from "./validateURL";
+import type { InvalidHeadersRule, ParsedHeaders, HeadersRule } from "./types";
+
+// Not strictly necessary to check for all protocols-like beginnings, since _technically_ that could be a legit header (e.g. name=http, value=://I'm a value).
+// But we're checking here since some people might be caught out and it'll help 99.9% of people who get it wrong.
+// We do the proper validation in `validateUrl` anyway :)
+const LINE_IS_PROBABLY_A_PATH = new RegExp(/^([^\s]+:\/\/|^\/)/);
+
+export function parseHeaders(input: string): ParsedHeaders {
+	const lines = input.split("\n");
+	const rules: HeadersRule[] = [];
+	const invalid: InvalidHeadersRule[] = [];
+
+	let rule: (HeadersRule & { line: string }) | undefined = undefined;
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i].trim();
+		if (line.length === 0 || line.startsWith("#")) continue;
+
+		if (line.length > MAX_LINE_LENGTH) {
+			invalid.push({
+				message: `Ignoring line ${
+					i + 1
+				} as it exceeds the maximum allowed length of ${MAX_LINE_LENGTH}.`,
+			});
+			continue;
+		}
+
+		if (LINE_IS_PROBABLY_A_PATH.test(line)) {
+			if (rules.length >= MAX_HEADER_RULES) {
+				invalid.push({
+					message: `Maximum number of rules supported is ${MAX_HEADER_RULES}. Skipping remaining ${
+						lines.length - i
+					} lines of file.`,
+				});
+				break;
+			}
+
+			if (rule) {
+				if (isValidRule(rule)) {
+					rules.push({
+						path: rule.path,
+						headers: rule.headers,
+						unsetHeaders: rule.unsetHeaders,
+					});
+				} else {
+					invalid.push({
+						line: rule.line,
+						lineNumber: i + 1,
+						message: "No headers specified",
+					});
+				}
+			}
+
+			const [path, pathError] = validateUrl(line);
+			if (pathError) {
+				invalid.push({
+					line,
+					lineNumber: i + 1,
+					message: pathError,
+				});
+				rule = undefined;
+				continue;
+			}
+
+			rule = {
+				path: path as string,
+				line,
+				headers: {},
+				unsetHeaders: [],
+			};
+			continue;
+		}
+
+		if (!line.includes(HEADER_SEPARATOR)) {
+			if (!rule) {
+				invalid.push({
+					line,
+					lineNumber: i + 1,
+					message: "Expected a path beginning with at least one forward-slash",
+				});
+			} else {
+				if (line.trim().startsWith(UNSET_OPERATOR)) {
+					rule.unsetHeaders.push(line.trim().replace(UNSET_OPERATOR, ""));
+				} else {
+					invalid.push({
+						line,
+						lineNumber: i + 1,
+						message:
+							"Expected a colon-separated header pair (e.g. name: value)",
+					});
+				}
+			}
+			continue;
+		}
+
+		const [rawName, ...rawValue] = line.split(HEADER_SEPARATOR);
+		const name = rawName.trim().toLowerCase();
+
+		if (name.includes(" ")) {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: "Header name cannot include spaces",
+			});
+			continue;
+		}
+
+		const value = rawValue.join(HEADER_SEPARATOR).trim();
+
+		if (name === "") {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: "No header name specified",
+			});
+			continue;
+		}
+
+		if (value === "") {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: "No header value specified",
+			});
+			continue;
+		}
+
+		if (!rule) {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: `Path should come before header (${name}: ${value})`,
+			});
+			continue;
+		}
+
+		const existingValues = rule.headers[name];
+		rule.headers[name] = existingValues ? `${existingValues}, ${value}` : value;
+	}
+
+	if (rule) {
+		if (isValidRule(rule)) {
+			rules.push({
+				path: rule.path,
+				headers: rule.headers,
+				unsetHeaders: rule.unsetHeaders,
+			});
+		} else {
+			invalid.push({ line: rule.line, message: "No headers specified" });
+		}
+	}
+
+	return {
+		rules,
+		invalid,
+	};
+}
+
+function isValidRule(rule: HeadersRule) {
+	return Object.keys(rule.headers).length > 0 || rule.unsetHeaders.length > 0;
+}

package/metadata-generator/parseRedirects.ts

@@ -0,0 +1,129 @@
+import {
+	MAX_LINE_LENGTH,
+	MAX_DYNAMIC_REDIRECT_RULES,
+	MAX_STATIC_REDIRECT_RULES,
+	PERMITTED_STATUS_CODES,
+	SPLAT_REGEX,
+	PLACEHOLDER_REGEX,
+} from "./constants";
+import { validateUrl } from "./validateURL";
+import type {
+	InvalidRedirectRule,
+	ParsedRedirects,
+	RedirectLine,
+	RedirectRule,
+} from "./types";
+
+export function parseRedirects(input: string): ParsedRedirects {
+	const lines = input.split("\n");
+	const rules: RedirectRule[] = [];
+	const seen_paths = new Set<string>();
+	const invalid: InvalidRedirectRule[] = [];
+
+	let staticRules = 0;
+	let dynamicRules = 0;
+	let canCreateStaticRule = true;
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i].trim();
+		if (line.length === 0 || line.startsWith("#")) continue;
+
+		if (line.length > MAX_LINE_LENGTH) {
+			invalid.push({
+				message: `Ignoring line ${
+					i + 1
+				} as it exceeds the maximum allowed length of ${MAX_LINE_LENGTH}.`,
+			});
+			continue;
+		}
+
+		const tokens = line.split(/\s+/);
+
+		if (tokens.length < 2 || tokens.length > 3) {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: `Expected exactly 2 or 3 whitespace-separated tokens. Got ${tokens.length}.`,
+			});
+			continue;
+		}
+
+		const [str_from, str_to, str_status = "302"] = tokens as RedirectLine;
+
+		const fromResult = validateUrl(str_from, true, false, false);
+		if (fromResult[0] === undefined) {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: fromResult[1],
+			});
+			continue;
+		}
+		const from = fromResult[0];
+
+		if (
+			canCreateStaticRule &&
+			!from.match(SPLAT_REGEX) &&
+			!from.match(PLACEHOLDER_REGEX)
+		) {
+			staticRules += 1;
+
+			if (staticRules > MAX_STATIC_REDIRECT_RULES) {
+				invalid.push({
+					message: `Maximum number of static rules supported is ${MAX_STATIC_REDIRECT_RULES}. Skipping line.`,
+				});
+				continue;
+			}
+		} else {
+			dynamicRules += 1;
+			canCreateStaticRule = false;
+
+			if (dynamicRules > MAX_DYNAMIC_REDIRECT_RULES) {
+				invalid.push({
+					message: `Maximum number of dynamic rules supported is ${MAX_DYNAMIC_REDIRECT_RULES}. Skipping remaining ${
+						lines.length - i
+					} lines of file.`,
+				});
+				break;
+			}
+		}
+
+		const toResult = validateUrl(str_to, false, true, true);
+		if (toResult[0] === undefined) {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: toResult[1],
+			});
+			continue;
+		}
+		const to = toResult[0];
+
+		const status = Number(str_status);
+		if (isNaN(status) || !PERMITTED_STATUS_CODES.has(status)) {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: `Valid status codes are 301, 302 (default), 303, 307, or 308. Got ${str_status}.`,
+			});
+			continue;
+		}
+
+		if (seen_paths.has(from)) {
+			invalid.push({
+				line,
+				lineNumber: i + 1,
+				message: `Ignoring duplicate rule for path ${from}.`,
+			});
+			continue;
+		}
+		seen_paths.add(from);
+
+		rules.push({ from, to, status, lineNumber: i + 1 });
+	}
+
+	return {
+		rules,
+		invalid,
+	};
+}

package/metadata-generator/types.ts

@@ -0,0 +1,90 @@
+/* REDIRECT PARSING TYPES */
+
+export type RedirectLine = [from: string, to: string, status?: number];
+export type RedirectRule = {
+	from: string;
+	to: string;
+	status: number;
+	lineNumber: number;
+};
+
+export type Headers = Record<string, string>;
+export type HeadersRule = {
+	path: string;
+	headers: Headers;
+	unsetHeaders: string[];
+};
+
+export type InvalidRedirectRule = {
+	line?: string;
+	lineNumber?: number;
+	message: string;
+};
+
+export type InvalidHeadersRule = {
+	line?: string;
+	lineNumber?: number;
+	message: string;
+};
+
+export type ParsedRedirects = {
+	invalid: InvalidRedirectRule[];
+	rules: RedirectRule[];
+};
+
+/** Parsed redirects and input file */
+export type ParsedRedirectsWithFile = {
+	parsedRedirects?: ParsedRedirects;
+	file?: string;
+};
+
+export type ParsedHeaders = {
+	invalid: InvalidHeadersRule[];
+	rules: HeadersRule[];
+};
+
+/** Parsed headers and input file */
+export type ParsedHeadersWithFile = {
+	parsedHeaders?: ParsedHeaders;
+	file?: string;
+};
+
+/* METADATA TYPES*/
+
+export type MetadataRedirectEntry = {
+	status: number;
+	to: string;
+};
+
+export type MetadataRedirects = {
+	[path: string]: MetadataRedirectEntry;
+};
+
+export type MetadataHeaders = {
+	[path: string]: MetadataHeaderEntry;
+};
+
+export type MetadataHeaderEntry = {
+	set?: Record<string, string>;
+	unset?: Array<string>;
+};
+
+export type Metadata = {
+	redirects?: {
+		version: number;
+		staticRules: MetadataRedirects;
+		rules: MetadataRedirects;
+	};
+	headers?: {
+		version: number;
+		rules: MetadataHeaders;
+	};
+	analytics?: {
+		version: number;
+		token: string;
+	};
+	deploymentId?: string;
+	failOpen?: boolean;
+};
+
+export type Logger = (message: string) => void;

package/metadata-generator/validateURL.ts

@@ -0,0 +1,57 @@
+export const extractPathname = (
+	path = "/",
+	includeSearch: boolean,
+	includeHash: boolean
+): string => {
+	if (!path.startsWith("/")) path = `/${path}`;
+	const url = new URL(`//${path}`, "relative://");
+	return `${url.pathname}${includeSearch ? url.search : ""}${
+		includeHash ? url.hash : ""
+	}`;
+};
+
+const URL_REGEX = /^https:\/\/+(?<host>[^/]+)\/?(?<path>.*)/;
+const PATH_REGEX = /^\//;
+
+export const validateUrl = (
+	token: string,
+	onlyRelative = false,
+	includeSearch = false,
+	includeHash = false
+): [undefined, string] | [string, undefined] => {
+	const host = URL_REGEX.exec(token);
+	if (host && host.groups && host.groups.host) {
+		if (onlyRelative)
+			return [
+				undefined,
+				`Only relative URLs are allowed. Skipping absolute URL ${token}.`,
+			];
+
+		return [
+			`https://${host.groups.host}${extractPathname(
+				host.groups.path,
+				includeSearch,
+				includeHash
+			)}`,
+			undefined,
+		];
+	} else {
+		if (!token.startsWith("/") && onlyRelative) token = `/${token}`;
+
+		const path = PATH_REGEX.exec(token);
+		if (path) {
+			try {
+				return [extractPathname(token, includeSearch, includeHash), undefined];
+			} catch {
+				return [undefined, `Error parsing URL segment ${token}. Skipping.`];
+			}
+		}
+	}
+
+	return [
+		undefined,
+		onlyRelative
+			? "URLs should begin with a forward-slash."
+			: 'URLs should either be relative (e.g. begin with a forward-slash), or use HTTPS (e.g. begin with "https://").',
+	];
+};

package/package.json

@@ -0,0 +1,63 @@
+{
+	"name": "@cloudflare/pages-shared",
+	"version": "0.0.0-ship-js",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/cloudflare/workers-sdk.git",
+		"directory": "packages/pages-shared"
+	},
+	"exports": {
+		"./asset-server/": "./dist/asset-server/",
+		"./environment-polyfills/": "./dist/environment-polyfills/",
+		"./metadata-generator/": "./dist/metadata-generator/"
+	},
+	"files": [
+		"dist/**/*",
+		"tsconfig.json",
+		"asset-server/**/*",
+		"environment-polyfills/**/*",
+		"metadata-generator/**/*"
+	],
+	"scripts": {
+		"build": "tsc",
+		"check:type": "tsc",
+		"test": "jest --forceExit",
+		"test:ci": "jest --forceExit"
+	},
+	"jest": {
+		"coverageReporters": [
+			"json",
+			"html",
+			"text",
+			"cobertura"
+		],
+		"restoreMocks": true,
+		"setupFilesAfterEnv": [
+			"<rootDir>/__tests__/jest.setup.ts"
+		],
+		"testRegex": ".*.(test|spec)\\.[jt]sx?$",
+		"testTimeout": 30000,
+		"transform": {
+			"^.+\\.c?(t|j)sx?$": [
+				"esbuild-jest",
+				{
+					"sourcemap": true
+				}
+			]
+		},
+		"transformIgnorePatterns": [
+			"node_modules/(?!find-up|locate-path|p-locate|p-limit|p-timeout|p-queue|yocto-queue|path-exists|execa|strip-final-newline|npm-run-path|path-key|onetime|mimic-fn|human-signals|is-stream|get-port|supports-color|pretty-bytes)"
+		]
+	},
+	"dependencies": {
+		"@miniflare/core": "2.13.0"
+	},
+	"devDependencies": {
+		"@miniflare/cache": "2.13.0",
+		"@miniflare/html-rewriter": "2.13.0",
+		"@types/service-worker-mock": "^2.0.1",
+		"concurrently": "^7.3.0",
+		"glob": "^8.0.3",
+		"service-worker-mock": "^2.0.5"
+	}
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"noEmit": false,
+		"declaration": true,
+		"declarationMap": true,
+		"outDir": "dist",
+		"module": "CommonJS",
+		"types": ["jest"]
+	},
+	"include": ["**/*.ts"],
+	"exclude": ["dist/**/*"]
+}