normalize-url 8.1.1 → 9.0.0

package/index.d.ts

@@ -4,6 +4,26 @@ export type Options = {
 	*/
 	readonly defaultProtocol?: 'https' | 'http';
 
+	/**
+	Protocols to normalize in addition to the built-in ones (`https`, `http`, `file`, `data`).
+
+	Useful for HTTP-like custom protocols such as Electron schemes or app-specific protocols.
+
+	The protocols should be specified without `:`.
+
+	@default undefined
+
+	@example
+	```
+	normalizeUrl('sindre://www.sorhus.com', {customProtocols: ['sindre']});
+	//=> 'sindre://sorhus.com'
+
+	normalizeUrl('sindre://www.sorhus.com/foo/', {customProtocols: ['sindre']});
+	//=> 'sindre://sorhus.com/foo'
+	```
+	*/
+	readonly customProtocols?: readonly string[];
+
 	/**
 	Prepends `defaultProtocol` to the URL if it's protocol-relative.
 
@@ -147,6 +167,8 @@ export type Options = {
 	/**
 	Removes query parameters that matches any of the provided strings or regexes.
 
+	Global and sticky regex flags are stripped.
+
 	@default [/^utm_\w+/i]
 
 	@example
@@ -182,6 +204,8 @@ export type Options = {
 
 	__Note__: It overrides the `removeQueryParameters` option.
 
+	Global and sticky regex flags are stripped.
+
 	@default undefined
 
 	@example
@@ -233,8 +257,11 @@ export type Options = {
 
 	/**
 	Removes the default directory index file from path that matches any of the provided strings or regexes.
+
 	When `true`, the regex `/^index\.[a-z]+$/` is used.
 
+	Global and sticky regex flags are stripped.
+
 	@default false
 
 	@example
@@ -279,6 +306,26 @@ export type Options = {
 	*/
 	readonly sortQueryParameters?: boolean;
 
+	/**
+	Controls how query parameters with empty values are formatted.
+
+	- `'preserve'` - Keep the original format (`?key` stays `?key`, `?key=` stays `?key=`). If the same key appears with both formats (`?a&a=`), all instances will use the format without `=`.
+	- `'always'` - Always include `=` for empty values (`?key` becomes `?key=`)
+	- `'never'` - Never include `=` for empty values (`?key=` becomes `?key`)
+
+	@default 'preserve'
+
+	@example
+	```
+	normalizeUrl('www.sindresorhus.com?a&b=', {emptyQueryValue: 'always'});
+	//=> 'http://sindresorhus.com/?a=&b='
+
+	normalizeUrl('www.sindresorhus.com?a&b=', {emptyQueryValue: 'never'});
+	//=> 'http://sindresorhus.com/?a&b'
+	```
+	*/
+	readonly emptyQueryValue?: 'preserve' | 'always' | 'never';
+
 	/**
 	Removes the entire URL path, leaving only the domain.
 
@@ -322,7 +369,7 @@ export type Options = {
 /**
 [Normalize](https://en.wikipedia.org/wiki/URL_normalization) a URL.
 
-URLs with custom protocols are not normalized and just passed through by default. Supported protocols are: `https`, `http`, `file`, and `data`.
+URLs with custom protocols are not normalized and just passed through by default. Supported protocols are: `https`, `http`, `file`, and `data`. Use the `customProtocols` option to add support for additional protocols.
 
 Human-friendly URLs with basic auth (for example, `user:password@sindresorhus.com`) are not handled because basic auth conflicts with custom protocols. [Basic auth URLs are also deprecated.](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#access_using_credentials_in_the_url)
 

package/index.js

@@ -2,7 +2,23 @@
 const DATA_URL_DEFAULT_MIME_TYPE = 'text/plain';
 const DATA_URL_DEFAULT_CHARSET = 'us-ascii';
 
-const testParameter = (name, filters) => filters.some(filter => filter instanceof RegExp ? filter.test(name) : filter === name);
+const encodedReservedCharactersPattern = '%(?:3A|2F|3F|23|5B|5D|40|21|24|26|27|28|29|2A|2B|2C|3B|3D)';
+const temporaryEncodedReservedTokenBase = '__normalize_url_encoded_reserved__';
+const temporaryEncodedReservedTokenPattern = /__normalize_url_encoded_reserved__(\d+)__/g;
+const hasEncodedReservedCharactersRegex = new RegExp(encodedReservedCharactersPattern, 'i');
+const encodedReservedCharactersRegex = new RegExp(encodedReservedCharactersPattern, 'gi');
+
+const testParameter = (name, filters) => Array.isArray(filters) && filters.some(filter => {
+	if (filter instanceof RegExp) {
+		if (filter.flags.includes('g') || filter.flags.includes('y')) {
+			return new RegExp(filter.source, filter.flags.replaceAll(/[gy]/g, '')).test(name);
+		}
+
+		return filter.test(name);
+	}
+
+	return filter === name;
+});
 
 const supportedProtocols = new Set([
 	'https:',
@@ -10,18 +26,157 @@ const supportedProtocols = new Set([
 	'file:',
 ]);
 
-const hasCustomProtocol = urlString => {
+const normalizeCustomProtocolOption = protocol => {
+	if (typeof protocol !== 'string') {
+		return undefined;
+	}
+
+	const normalizedProtocol = protocol.trim().toLowerCase().replace(/:$/, '');
+	return normalizedProtocol === '' ? undefined : `${normalizedProtocol}:`;
+};
+
+const getCustomProtocol = urlString => {
 	try {
 		const {protocol} = new URL(urlString);
+		const hasAuthority = urlString.slice(0, protocol.length + 2).toLowerCase() === `${protocol}//`;
+
+		if (protocol.endsWith(':')
+			&& (!protocol.includes('.') || hasAuthority)
+			&& !supportedProtocols.has(protocol)) {
+			return protocol;
+		}
+	} catch {}
 
-		return protocol.endsWith(':')
-			&& !protocol.includes('.')
-			&& !supportedProtocols.has(protocol);
+	return undefined;
+};
+
+const decodeQueryKey = value => {
+	try {
+		return decodeURIComponent(value.replaceAll('+', '%20'));
 	} catch {
-		return false;
+		// Match URLSearchParams behavior for malformed percent-encoding.
+		return new URLSearchParams(`${value}=`).keys().next().value;
 	}
 };
 
+const getKeysWithoutEquals = search => {
+	const keys = new Set();
+	if (!search) {
+		return keys;
+	}
+
+	for (const part of search.slice(1).split('&')) {
+		if (part && !part.includes('=')) {
+			keys.add(decodeQueryKey(part));
+		}
+	}
+
+	return keys;
+};
+
+const getTemporaryEncodedReservedTokenPrefix = search => {
+	let decodedSearch = search;
+
+	try {
+		decodedSearch = decodeURIComponent(search);
+	} catch {
+		decodedSearch = new URLSearchParams(search).toString();
+	}
+
+	const getUsedTokenIndexes = value => {
+		const indexes = new Set();
+
+		for (const match of value.matchAll(temporaryEncodedReservedTokenPattern)) {
+			indexes.add(Number.parseInt(match[1], 10));
+		}
+
+		return indexes;
+	};
+
+	const usedTokenIndexes = getUsedTokenIndexes(search);
+	for (const tokenIndex of getUsedTokenIndexes(decodedSearch)) {
+		usedTokenIndexes.add(tokenIndex);
+	}
+
+	let tokenIndex = 0;
+	while (usedTokenIndexes.has(tokenIndex)) {
+		tokenIndex++;
+	}
+
+	return `${temporaryEncodedReservedTokenBase}${tokenIndex}__`;
+};
+
+const sortSearchParameters = (searchParameters, encodedReservedTokenRegex) => {
+	if (!encodedReservedTokenRegex) {
+		searchParameters.sort();
+		return searchParameters.toString();
+	}
+
+	const getSortableKey = key => key.replace(encodedReservedTokenRegex, (_, hexCode) => String.fromCodePoint(Number.parseInt(hexCode, 16)));
+	const entries = [...searchParameters.entries()];
+	entries.sort(([leftKey], [rightKey]) => {
+		const left = getSortableKey(leftKey);
+		const right = getSortableKey(rightKey);
+		return left < right ? -1 : (left > right ? 1 : 0);
+	});
+
+	return new URLSearchParams(entries).toString();
+};
+
+const decodeReservedTokens = (value, encodedReservedTokenRegex) => {
+	if (!encodedReservedTokenRegex) {
+		return value;
+	}
+
+	return value.replace(encodedReservedTokenRegex, (_, hexCode) => String.fromCodePoint(Number.parseInt(hexCode, 16)));
+};
+
+const normalizeEmptyQueryParameters = (search, emptyQueryValue, originalSearch) => {
+	const isAlways = emptyQueryValue === 'always';
+	const isNever = emptyQueryValue === 'never';
+	const keysWithoutEquals = (isAlways || isNever) ? undefined : getKeysWithoutEquals(originalSearch);
+
+	const normalizeKey = key => key.replaceAll('+', '%20');
+	const formatEmptyValue = normalizedKey => {
+		if (isAlways) {
+			return `${normalizedKey}=`;
+		}
+
+		if (isNever) {
+			return normalizedKey;
+		}
+
+		return keysWithoutEquals.has(decodeQueryKey(normalizedKey)) ? normalizedKey : `${normalizedKey}=`;
+	};
+
+	const normalizeParameter = parameter => {
+		const equalIndex = parameter.indexOf('=');
+
+		if (equalIndex === -1) {
+			// Normalize + to %20 (+ means space in query strings)
+			return formatEmptyValue(normalizeKey(parameter));
+		}
+
+		const key = parameter.slice(0, equalIndex);
+		const value = parameter.slice(equalIndex + 1);
+
+		if (value === '') {
+			if (key === '') {
+				return '=';
+			}
+
+			// Normalize + to %20 (+ means space in query strings)
+			return formatEmptyValue(normalizeKey(key));
+		}
+
+		// Normalize + to %20 in key.
+		return `${normalizeKey(key)}=${value}`;
+	};
+
+	const parameters = search.slice(1).split('&').filter(Boolean);
+	return parameters.length === 0 ? '' : `?${parameters.map(x => normalizeParameter(x)).join('&')}`;
+};
+
 const normalizeDataURL = (urlString, {stripHash}) => {
 	const match = /^data:(?<type>[^,]*?),(?<data>[^#]*?)(?:#(?<hash>.*))?$/.exec(urlString);
 
@@ -38,7 +193,7 @@ const normalizeDataURL = (urlString, {stripHash}) => {
 	}
 
 	// Lowercase MIME type
-	const mimeType = mediaType.shift()?.toLowerCase() ?? '';
+	const mimeType = mediaType.shift().toLowerCase();
 	const attributes = mediaType
 		.map(attribute => {
 			let [key, value = ''] = attribute.split('=').map(string => string.trim());
@@ -88,6 +243,7 @@ export default function normalizeUrl(urlString, options) {
 		sortQueryParameters: true,
 		removePath: false,
 		transformPath: false,
+		emptyQueryValue: 'preserve',
 		...options,
 	};
 
@@ -103,7 +259,13 @@ export default function normalizeUrl(urlString, options) {
 		return normalizeDataURL(urlString, options);
 	}
 
-	if (hasCustomProtocol(urlString)) {
+	const customProtocols = Array.isArray(options.customProtocols) ? options.customProtocols : [];
+	const normalizedCustomProtocols = new Set(customProtocols
+		.map(protocol => normalizeCustomProtocolOption(protocol))
+		.filter(Boolean));
+
+	const customProtocol = getCustomProtocol(urlString);
+	if (customProtocol && !normalizedCustomProtocols.has(customProtocol)) {
 		return urlString;
 	}
 
@@ -111,7 +273,7 @@ export default function normalizeUrl(urlString, options) {
 	const isRelativeUrl = !hasRelativeProtocol && /^\.*\//.test(urlString);
 
 	// Prepend protocol
-	if (!isRelativeUrl) {
+	if (!isRelativeUrl && !customProtocol) {
 		urlString = urlString.replace(/^(?!(?:\w+:)?\/\/)|^\/\//, options.defaultProtocol);
 	}
 
@@ -166,13 +328,13 @@ export default function normalizeUrl(urlString, options) {
 			const protocolAtIndex = match.index;
 			const intermediate = urlObject.pathname.slice(lastIndex, protocolAtIndex);
 
-			result += intermediate.replace(/\/{2,}/g, '/');
+			result += intermediate.replaceAll(/\/{2,}/g, '/');
 			result += protocol;
 			lastIndex = protocolAtIndex + protocol.length;
 		}
 
-		const remnant = urlObject.pathname.slice(lastIndex, urlObject.pathname.length);
-		result += remnant.replace(/\/{2,}/g, '/');
+		const remnant = urlObject.pathname.slice(lastIndex);
+		result += remnant.replaceAll(/\/{2,}/g, '/');
 
 		urlObject.pathname = result;
 	}
@@ -180,7 +342,7 @@ export default function normalizeUrl(urlString, options) {
 	// Decode URI octets
 	if (urlObject.pathname) {
 		try {
-			urlObject.pathname = decodeURI(urlObject.pathname).replace(/\\/g, '%5C');
+			urlObject.pathname = decodeURI(urlObject.pathname).replaceAll('\\', '%5C');
 		} catch {}
 	}
 
@@ -225,49 +387,61 @@ export default function normalizeUrl(urlString, options) {
 		}
 	}
 
+	// Capture original query params format before any searchParams modifications
+	const originalSearch = urlObject.search;
+	let encodedReservedTokenRegex;
+
+	if (options.sortQueryParameters && hasEncodedReservedCharactersRegex.test(originalSearch)) {
+		const encodedReservedTokenPrefix = getTemporaryEncodedReservedTokenPrefix(originalSearch);
+		urlObject.search = originalSearch.replaceAll(encodedReservedCharactersRegex, match => `${encodedReservedTokenPrefix}${match.slice(1).toUpperCase()}`);
+		encodedReservedTokenRegex = new RegExp(`${encodedReservedTokenPrefix}([0-9A-F]{2})`, 'g');
+	}
+
+	const hasKeepQueryParameters = Array.isArray(options.keepQueryParameters);
+	const {searchParams} = urlObject;
+
 	// Remove query unwanted parameters
-	if (Array.isArray(options.removeQueryParameters)) {
+	if (!hasKeepQueryParameters && Array.isArray(options.removeQueryParameters) && options.removeQueryParameters.length > 0) {
 		// eslint-disable-next-line unicorn/no-useless-spread -- We are intentionally spreading to get a copy.
-		for (const key of [...urlObject.searchParams.keys()]) {
-			if (testParameter(key, options.removeQueryParameters)) {
-				urlObject.searchParams.delete(key);
+		for (const key of [...searchParams.keys()]) {
+			if (testParameter(decodeReservedTokens(key, encodedReservedTokenRegex), options.removeQueryParameters)) {
+				searchParams.delete(key);
 			}
 		}
 	}
 
-	if (!Array.isArray(options.keepQueryParameters) && options.removeQueryParameters === true) {
+	if (!hasKeepQueryParameters && options.removeQueryParameters === true) {
 		urlObject.search = '';
 	}
 
 	// Keep wanted query parameters
-	if (Array.isArray(options.keepQueryParameters) && options.keepQueryParameters.length > 0) {
+	if (hasKeepQueryParameters && options.keepQueryParameters.length > 0) {
 		// eslint-disable-next-line unicorn/no-useless-spread -- We are intentionally spreading to get a copy.
-		for (const key of [...urlObject.searchParams.keys()]) {
-			if (!testParameter(key, options.keepQueryParameters)) {
-				urlObject.searchParams.delete(key);
+		for (const key of [...searchParams.keys()]) {
+			if (!testParameter(decodeReservedTokens(key, encodedReservedTokenRegex), options.keepQueryParameters)) {
+				searchParams.delete(key);
 			}
 		}
+	} else if (hasKeepQueryParameters) {
+		urlObject.search = '';
 	}
 
 	// Sort query parameters
 	if (options.sortQueryParameters) {
-		const originalSearch = urlObject.search;
-		urlObject.searchParams.sort();
+		urlObject.search = sortSearchParameters(urlObject.searchParams, encodedReservedTokenRegex);
 
-		// Calling `.sort()` encodes the search parameters, so we need to decode them again.
-		try {
-			urlObject.search = decodeURIComponent(urlObject.search);
-		} catch {}
+		// Sorting and serializing encode the search parameters, so we need to decode them again.
+		// Protect &%#? and %2B from decoding (would break URL structure or change meaning) by double-encoding them first.
+		urlObject.search = decodeURIComponent(urlObject.search.replaceAll(/%(?:26|23|3f|25|2b)/gi, match => `%25${match.slice(1)}`));
 
-		// Fix parameters that originally had no equals sign but got one added by URLSearchParams
-		const partsWithoutEquals = originalSearch.slice(1).split('&').filter(p => p && !p.includes('='));
-		for (const part of partsWithoutEquals) {
-			const decoded = decodeURIComponent(part);
-			// Only replace at word boundaries to avoid partial matches
-			urlObject.search = urlObject.search.replace(`?${decoded}=`, `?${decoded}`).replace(`&${decoded}=`, `&${decoded}`);
+		if (encodedReservedTokenRegex) {
+			urlObject.search = urlObject.search.replace(encodedReservedTokenRegex, '%$1');
 		}
 	}
 
+	// Normalize empty query parameter values
+	urlObject.search = normalizeEmptyQueryParameters(urlObject.search, options.emptyQueryValue, originalSearch);
+
 	if (options.removeTrailingSlash) {
 		urlObject.pathname = urlObject.pathname.replace(/\/$/, '');
 	}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "normalize-url",
-	"version": "8.1.1",
+	"version": "9.0.0",
 	"description": "Normalize a URL",
 	"license": "MIT",
 	"repository": "sindresorhus/normalize-url",
@@ -17,11 +17,10 @@
 	},
 	"sideEffects": false,
 	"engines": {
-		"node": ">=14.16"
+		"node": ">=20"
 	},
 	"scripts": {
-		"//test": "xo && c8 ava && tsd",
-		"test": "c8 ava && tsd"
+		"test": "xo && ava && tsd"
 	},
 	"files": [
 		"index.js",
@@ -43,10 +42,9 @@
 		"canonical"
 	],
 	"devDependencies": {
-		"ava": "^5.0.1",
-		"c8": "^7.12.0",
-		"tsd": "^0.24.1",
-		"xo": "^0.52.4"
+		"ava": "^6.4.1",
+		"tsd": "^0.33.0",
+		"xo": "^1.2.3"
 	},
 	"c8": {
 		"reporter": [

package/readme.md

@@ -4,7 +4,8 @@
 
 Useful when you need to display, store, deduplicate, sort, compare, etc, URLs.
 
-**Note:** This package does **not** do URL sanitization. [Garbage in, garbage out.](https://en.wikipedia.org/wiki/Garbage_in,_garbage_out) If you use this in a server context and accept URLs as user input, it's up to you to protect against invalid URLs, [path traversal attacks](https://owasp.org/www-community/attacks/Path_Traversal), etc.
+> [!NOTE]
+> This package does **not** do URL sanitization. [Garbage in, garbage out.](https://en.wikipedia.org/wiki/Garbage_in,_garbage_out) If you use this in a server context and accept URLs as user input, it's up to you to protect against invalid URLs, [path traversal attacks](https://owasp.org/www-community/attacks/Path_Traversal), etc.
 
 ## Install
 
@@ -28,7 +29,7 @@ normalizeUrl('//www.sindresorhus.com:80/../baz?b=bar&a=foo');
 
 ### normalizeUrl(url, options?)
 
-URLs with custom protocols are not normalized and just passed through by default. Supported protocols are: `https`, `http`, `file`, and `data`.
+URLs with custom protocols are not normalized and just passed through by default. Supported protocols are: `https`, `http`, `file`, and `data`. Use the [`customProtocols`](#customprotocols) option to add support for additional protocols.
 
 Human-friendly URLs with basic auth (for example, `user:password@sindresorhus.com`) are not handled because basic auth conflicts with custom protocols. [Basic auth URLs are also deprecated.](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#access_using_credentials_in_the_url)
 
@@ -48,6 +49,25 @@ Type: `string`\
 Default: `'http'`\
 Values: `'https' | 'http'`
 
+##### customProtocols
+
+Type: `string[]`\
+Default: `undefined`
+
+Protocols to normalize in addition to the built-in ones (`https`, `http`, `file`, `data`).
+
+Useful for HTTP-like custom protocols such as Electron schemes or app-specific protocols.
+
+The protocols should be specified without `:`.
+
+```js
+normalizeUrl('sindre://www.sorhus.com', {customProtocols: ['sindre']});
+//=> 'sindre://sorhus.com'
+
+normalizeUrl('sindre://www.sorhus.com/foo/', {customProtocols: ['sindre']});
+//=> 'sindre://sorhus.com/foo'
+```
+
 ##### normalizeProtocol
 
 Type: `boolean`\
@@ -187,6 +207,8 @@ Default: `[/^utm_\w+/i]`
 
 Remove query parameters that matches any of the provided strings or regexes.
 
+Global and sticky regex flags are stripped.
+
 ```js
 normalizeUrl('www.sindresorhus.com?foo=bar&ref=test_ref', {
 	removeQueryParameters: ['ref']
@@ -221,6 +243,8 @@ Keeps only query parameters that matches any of the provided strings or regexes.
 
 **Note:** It overrides the `removeQueryParameters` option.
 
+Global and sticky regex flags are stripped.
+
 ```js
 normalizeUrl('https://sindresorhus.com?foo=bar&ref=unicorn', {
 	keepQueryParameters: ['ref']
@@ -268,7 +292,11 @@ normalizeUrl('https://sindresorhus.com/', {removeSingleSlash: false});
 Type: `boolean | Array<RegExp | string>`\
 Default: `false`
 
-Removes the default directory index file from path that matches any of the provided strings or regexes. When `true`, the regex `/^index\.[a-z]+$/` is used.
+Removes the default directory index file from path that matches any of the provided strings or regexes.
+
+When `true`, the regex `/^index\.[a-z]+$/` is used.
+
+Global and sticky regex flags are stripped.
 
 ```js
 normalizeUrl('www.sindresorhus.com/foo/default.php', {
@@ -307,6 +335,26 @@ normalizeUrl('www.sindresorhus.com?b=two&a=one&c=three', {
 //=> 'http://sindresorhus.com/?b=two&a=one&c=three'
 ```
 
+##### emptyQueryValue
+
+Type: `string`\
+Default: `'preserve'`\
+Values: `'preserve' | 'always' | 'never'`
+
+Controls how query parameters with empty values are formatted.
+
+- `'preserve'` - Keep the original format (`?key` stays `?key`, `?key=` stays `?key=`). If the same key appears with both formats (`?a&a=`), all instances will use the format without `=`.
+- `'always'` - Always include `=` for empty values (`?key` becomes `?key=`)
+- `'never'` - Never include `=` for empty values (`?key=` becomes `?key`)
+
+```js
+normalizeUrl('www.sindresorhus.com?a&b=', {emptyQueryValue: 'always'});
+//=> 'http://sindresorhus.com/?a=&b='
+
+normalizeUrl('www.sindresorhus.com?a&b=', {emptyQueryValue: 'never'});
+//=> 'http://sindresorhus.com/?a&b'
+```
+
 ##### removePath
 
 Type: `boolean`\