fast-uri 2.3.1 → 2.4.1

package/README.md

@@ -15,6 +15,8 @@ Dependency free RFC 3986 URI toolbox.
 
 All of the above functions can accept an additional options argument that is an object that can contain one or more of the following properties:
 
+Malformed authorities and out-of-range ports are reported through the parsed component's `error` field. `normalize()` leaves malformed string inputs unchanged, and `equal()` returns `false` when either string input is malformed.
+
 *	`scheme` (string)
 	Indicates the scheme that the URI should be treated as, overriding the URI's normal scheme parsing behavior.
 
@@ -69,6 +71,17 @@ uri.resolve("uri://a/b/c/d?q", "../../g")
 "uri://a/g"
 ```
 
+### Normalize
+
+```js
+const uri = require('fast-uri')
+uri.normalize('http://example.com/a%2Fb')
+// Output
+"http://example.com/a%2Fb"
+```
+
+Reserved path escapes such as `%2F` and `%2E` are preserved as path data during normalization and comparison.
+
 ### Equal
 
 ```js

package/index.js

@@ -1,11 +1,11 @@
 'use strict'
 
-const { normalizeIPv6, normalizeIPv4, removeDotSegments, recomposeAuthority, normalizeComponentEncoding } = require('./lib/utils')
+const { normalizeIPv6, normalizeIPv4, removeDotSegments, recomposeAuthority, normalizePercentEncoding, normalizePathEncoding, escapePreservingEscapes, reescapeHostDelimiters } = require('./lib/utils')
 const SCHEMES = require('./lib/schemes')
 
 function normalize (uri, options) {
   if (typeof uri === 'string') {
-    uri = serialize(parse(uri, options), options)
+    uri = normalizeString(uri, options)
   } else if (typeof uri === 'object') {
     uri = parse(serialize(uri, options), options)
   }
@@ -79,21 +79,10 @@ function resolveComponents (base, relative, options, skipNormalization) {
 }
 
 function equal (uriA, uriB, options) {
-  if (typeof uriA === 'string') {
-    uriA = unescape(uriA)
-    uriA = serialize(normalizeComponentEncoding(parse(uriA, options), true), { ...options, skipEscape: true })
-  } else if (typeof uriA === 'object') {
-    uriA = serialize(normalizeComponentEncoding(uriA, true), { ...options, skipEscape: true })
-  }
-
-  if (typeof uriB === 'string') {
-    uriB = unescape(uriB)
-    uriB = serialize(normalizeComponentEncoding(parse(uriB, options), true), { ...options, skipEscape: true })
-  } else if (typeof uriB === 'object') {
-    uriB = serialize(normalizeComponentEncoding(uriB, true), { ...options, skipEscape: true })
-  }
+  const normalizedA = normalizeComparableURI(uriA, options)
+  const normalizedB = normalizeComparableURI(uriB, options)
 
-  return uriA.toLowerCase() === uriB.toLowerCase()
+  return normalizedA !== undefined && normalizedB !== undefined && normalizedA.toLowerCase() === normalizedB.toLowerCase()
 }
 
 function serialize (cmpts, opts) {
@@ -124,13 +113,13 @@ function serialize (cmpts, opts) {
 
   if (components.path !== undefined) {
     if (!options.skipEscape) {
-      components.path = escape(components.path)
+      components.path = escapePreservingEscapes(components.path)
 
       if (components.scheme !== undefined) {
         components.path = components.path.split('%3A').join(':')
       }
     } else {
-      components.path = unescape(components.path)
+      components.path = normalizePercentEncoding(components.path)
     }
   }
 
@@ -192,7 +181,19 @@ function nonSimpleDomain (value) {
 
 const URI_PARSE = /^(?:([^#/:?]+):)?(?:\/\/((?:([^#/?@]*)@)?(\[[^#/?\]]+\]|[^#/:?]*)(?::(\d*))?))?([^#?]*)(?:\?([^#]*))?(?:#((?:.|[\n\r])*))?/u
 
-function parse (uri, opts) {
+function getParseError (parsed, matches) {
+  if (matches[2] !== undefined && parsed.path && parsed.path[0] !== '/') {
+    return 'URI path must start with "/" when authority is present.'
+  }
+
+  if (typeof parsed.port === 'number' && (parsed.port < 0 || parsed.port > 65535)) {
+    return 'URI port is malformed.'
+  }
+
+  return undefined
+}
+
+function parseWithStatus (uri, opts) {
   const options = Object.assign({}, opts)
   const parsed = {
     scheme: undefined,
@@ -204,6 +205,8 @@ function parse (uri, opts) {
     fragment: undefined
   }
   const gotEncoding = uri.indexOf('%') !== -1
+  let malformedAuthorityOrPort = false
+  let isIP = false
   if (options.reference === 'suffix') uri = (options.scheme ? options.scheme + ':' : '') + '//' + uri
 
   const matches = uri.match(URI_PARSE)
@@ -222,12 +225,22 @@ function parse (uri, opts) {
     if (isNaN(parsed.port)) {
       parsed.port = matches[5]
     }
+
+    const parseError = getParseError(parsed, matches)
+    if (parseError !== undefined) {
+      parsed.error = parsed.error || parseError
+      malformedAuthorityOrPort = true
+    }
+
     if (parsed.host) {
       const ipv4result = normalizeIPv4(parsed.host)
       if (ipv4result.isIPV4 === false) {
-        parsed.host = normalizeIPv6(ipv4result.host, { isIPV4: false }).host.toLowerCase()
+        const ipv6result = normalizeIPv6(ipv4result.host, { isIPV4: false })
+        parsed.host = ipv6result.host.toLowerCase()
+        isIP = ipv6result.isIPV6
       } else {
         parsed.host = ipv4result.host
+        isIP = true
       }
     }
     if (parsed.scheme === undefined && parsed.userinfo === undefined && parsed.host === undefined && parsed.port === undefined && !parsed.path && parsed.query === undefined) {
@@ -251,7 +264,7 @@ function parse (uri, opts) {
     // check if scheme can't handle IRIs
     if (!options.unicodeSupport && (!schemeHandler || !schemeHandler.unicodeSupport)) {
       // if host component is a domain name
-      if (parsed.host && (options.domainHost || (schemeHandler && schemeHandler.domainHost)) && nonSimpleDomain(parsed.host)) {
+      if (parsed.host && (options.domainHost || (schemeHandler && schemeHandler.domainHost)) && isIP === false && nonSimpleDomain(parsed.host)) {
         // convert Unicode IDN -> ASCII IDN
         try {
           parsed.host = URL.domainToASCII(parsed.host.toLowerCase())
@@ -270,10 +283,10 @@ function parse (uri, opts) {
         parsed.userinfo = unescape(parsed.userinfo)
       }
       if (gotEncoding && parsed.host !== undefined) {
-        parsed.host = unescape(parsed.host)
+        parsed.host = reescapeHostDelimiters(unescape(parsed.host), isIP)
       }
       if (parsed.path !== undefined && parsed.path.length) {
-        parsed.path = encodeURI(parsed.path)
+        parsed.path = normalizePathEncoding(parsed.path)
       }
       if (parsed.fragment !== undefined && parsed.fragment.length) {
         parsed.fragment = encodeURI(decodeURI(parsed.fragment))
@@ -287,10 +300,38 @@ function parse (uri, opts) {
   } else {
     parsed.error = parsed.error || 'URI can not be parsed.'
   }
-  return parsed
+  return { parsed, malformedAuthorityOrPort }
+}
+
+function parse (uri, opts) {
+  return parseWithStatus(uri, opts).parsed
+}
+
+function normalizeString (uri, opts) {
+  return normalizeStringWithStatus(uri, opts).normalized
+}
+
+function normalizeStringWithStatus (uri, opts) {
+  const { parsed, malformedAuthorityOrPort } = parseWithStatus(uri, opts)
+  return {
+    normalized: malformedAuthorityOrPort ? uri : serialize(parsed, opts),
+    malformedAuthorityOrPort
+  }
+}
+
+function normalizeComparableURI (uri, opts) {
+  if (typeof uri === 'string') {
+    const { normalized, malformedAuthorityOrPort } = normalizeStringWithStatus(uri, opts)
+    return malformedAuthorityOrPort ? undefined : normalized
+  }
+
+  if (typeof uri === 'object') {
+    return serialize(uri, opts)
+  }
 }
 
 const fastUri = {
+  SCHEMES,
   normalize,
   resolve,
   resolveComponents,

package/lib/utils.js

@@ -13,14 +13,20 @@ function normalizeIPv4 (host) {
   }
 }
 
-function stringToHexStripped (input) {
+/**
+ * @param {string[]} input
+ * @param {boolean} [keepZero=false]
+ * @returns {string|undefined}
+ */
+function stringArrayToHexStripped (input, keepZero = false) {
   let acc = ''
   let strip = true
   for (const c of input) {
-    if (c !== '0' && strip === true) strip = false
     if (HEX[c] === undefined) return undefined
+    if (c !== '0' && strip === true) strip = false
     if (!strip) acc += c
   }
+  if (keepZero && acc.length === 0) acc = '0'
   return acc
 }
 
@@ -36,7 +42,7 @@ function getIPV6 (input) {
   function consume () {
     if (buffer.length) {
       if (isZone === false) {
-        const hex = stringToHexStripped(buffer.join(''))
+        const hex = stringArrayToHexStripped(buffer)
         if (hex !== undefined) {
           address.push(hex)
         } else {
@@ -83,7 +89,7 @@ function getIPV6 (input) {
     } else if (endIpv6) {
       address.push(buffer.join(''))
     } else {
-      address.push(stringToHexStripped(buffer.join('')))
+      address.push(stringArrayToHexStripped(buffer))
     }
   }
   output.address = address.join('')
@@ -170,27 +176,131 @@ function removeDotSegments (input) {
   return output.join('')
 }
 
-function normalizeComponentEncoding (components, esc) {
-  const func = esc !== true ? escape : unescape
-  if (components.scheme !== undefined) {
-    components.scheme = func(components.scheme)
-  }
-  if (components.userinfo !== undefined) {
-    components.userinfo = func(components.userinfo)
-  }
-  if (components.host !== undefined) {
-    components.host = func(components.host)
+/**
+ * Re-escape RFC 3986 gen-delims that must not appear literally in the host.
+ * After the URI regex parses, these characters cannot be literal in the host
+ * field, so any that appear after decoding came from percent-encoding and
+ * must be restored to prevent authority structure changes.
+ *
+ * @param {string} host
+ * @param {boolean} isIP - true for IPv4/IPv6 hosts (skip colon re-escaping)
+ * @returns {string}
+ */
+const HOST_DELIMS = { '@': '%40', '/': '%2F', '?': '%3F', '#': '%23', ':': '%3A' }
+const HOST_DELIM_RE = /[@/?#:]/g
+const HOST_DELIM_NO_COLON_RE = /[@/?#]/g
+
+function reescapeHostDelimiters (host, isIP) {
+  const re = isIP ? HOST_DELIM_NO_COLON_RE : HOST_DELIM_RE
+  re.lastIndex = 0
+  return host.replace(re, (ch) => HOST_DELIMS[ch])
+}
+
+const isHexPair = RegExp.prototype.test.bind(/^[\da-f]{2}$/iu)
+const isUnreserved = RegExp.prototype.test.bind(/^[\da-z\-._~]$/iu)
+const isPathCharacter = RegExp.prototype.test.bind(/^[\da-z\-._~!$&'()*+,;=:@/]$/iu)
+
+/**
+ * Normalizes percent escapes and optionally decodes only unreserved ASCII bytes.
+ * Reserved delimiters such as `%2F` and `%2E` stay escaped.
+ *
+ * @param {string} input
+ * @param {boolean} [decodeUnreserved=false]
+ * @returns {string}
+ */
+function normalizePercentEncoding (input, decodeUnreserved = false) {
+  if (input.indexOf('%') === -1) {
+    return input
   }
-  if (components.path !== undefined) {
-    components.path = func(components.path)
+
+  let output = ''
+
+  for (let i = 0; i < input.length; i++) {
+    if (input[i] === '%' && i + 2 < input.length) {
+      const hex = input.slice(i + 1, i + 3)
+      if (isHexPair(hex)) {
+        const normalizedHex = hex.toUpperCase()
+        const decoded = String.fromCharCode(parseInt(normalizedHex, 16))
+
+        if (decodeUnreserved && isUnreserved(decoded)) {
+          output += decoded
+        } else {
+          output += '%' + normalizedHex
+        }
+
+        i += 2
+        continue
+      }
+    }
+
+    output += input[i]
   }
-  if (components.query !== undefined) {
-    components.query = func(components.query)
+
+  return output
+}
+
+/**
+ * Normalizes path data without turning reserved escapes into live path syntax.
+ * Valid escapes are uppercased, raw unsafe characters are escaped, and only
+ * unreserved bytes that are not `.` are decoded.
+ *
+ * @param {string} input
+ * @returns {string}
+ */
+function normalizePathEncoding (input) {
+  let output = ''
+
+  for (let i = 0; i < input.length; i++) {
+    if (input[i] === '%' && i + 2 < input.length) {
+      const hex = input.slice(i + 1, i + 3)
+      if (isHexPair(hex)) {
+        const normalizedHex = hex.toUpperCase()
+        const decoded = String.fromCharCode(parseInt(normalizedHex, 16))
+
+        if (decoded !== '.' && isUnreserved(decoded)) {
+          output += decoded
+        } else {
+          output += '%' + normalizedHex
+        }
+
+        i += 2
+        continue
+      }
+    }
+
+    if (isPathCharacter(input[i])) {
+      output += input[i]
+    } else {
+      output += escape(input[i])
+    }
   }
-  if (components.fragment !== undefined) {
-    components.fragment = func(components.fragment)
+
+  return output
+}
+
+/**
+ * Escapes a component while preserving existing valid percent escapes.
+ *
+ * @param {string} input
+ * @returns {string}
+ */
+function escapePreservingEscapes (input) {
+  let output = ''
+
+  for (let i = 0; i < input.length; i++) {
+    if (input[i] === '%' && i + 2 < input.length) {
+      const hex = input.slice(i + 1, i + 3)
+      if (isHexPair(hex)) {
+        output += '%' + hex.toUpperCase()
+        i += 2
+        continue
+      }
+    }
+
+    output += escape(input[i])
   }
-  return components
+
+  return output
 }
 
 function recomposeAuthority (components, options) {
@@ -212,7 +322,7 @@ function recomposeAuthority (components, options) {
       if (ipV6res.isIPV6 === true) {
         host = `[${ipV6res.escapedHost}]`
       } else {
-        host = components.host
+        host = reescapeHostDelimiters(host, false)
       }
     }
     uriTokens.push(host)
@@ -228,9 +338,12 @@ function recomposeAuthority (components, options) {
 
 module.exports = {
   recomposeAuthority,
-  normalizeComponentEncoding,
+  reescapeHostDelimiters,
+  normalizePercentEncoding,
+  normalizePathEncoding,
+  escapePreservingEscapes,
   removeDotSegments,
   normalizeIPv4,
   normalizeIPv6,
-  stringToHexStripped
+  stringArrayToHexStripped
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "fast-uri",
   "description": "Dependency free RFC 3986 URI toolbox",
-  "version": "2.3.1",
+  "version": "2.4.1",
   "main": "index.js",
   "type": "commonjs",
   "types": "types/index.d.ts",

package/test/compatibility.test.js

@@ -9,6 +9,7 @@ test('compatibility Parse', (t) => {
   const toParse = [
     '//www.g.com/error\n/bleh/bleh',
     'https://fastify.org',
+    '/definitions/Record%3Cstring%2CPerson%3E',
     '//10.10.10.10',
     '//10.10.000.10',
     '//[2001:db8::7%en0]',

package/test/security-normalization.test.js

@@ -0,0 +1,40 @@
+'use strict'
+
+const tap = require('tap')
+const test = tap.test
+const fastURI = require('..')
+
+test('parse preserves reserved path escapes as data', (t) => {
+  const components = fastURI.parse('http://example.com/a%2Fb/public/%2e%2e/admin')
+
+  t.equal(components.path, '/a%2Fb/public/%2E%2E/admin')
+  t.end()
+})
+
+test('normalize preserves percent-encoded path separators and dot segments', (t) => {
+  t.equal(
+    fastURI.normalize('http://example.com/public/%2e%2e/admin'),
+    'http://example.com/public/%2E%2E/admin'
+  )
+
+  t.equal(
+    fastURI.normalize('http://example.com/a%2Fb'),
+    'http://example.com/a%2Fb'
+  )
+
+  t.end()
+})
+
+test('equal does not treat reserved path escapes as live path syntax', (t) => {
+  t.equal(
+    fastURI.equal('http://example.com/public/%2e%2e/admin', 'http://example.com/admin', {}),
+    false
+  )
+
+  t.equal(
+    fastURI.equal('http://example.com/a%2Fb', 'http://example.com/a/b', {}),
+    false
+  )
+
+  t.end()
+})

package/test/security.test.js

@@ -0,0 +1,134 @@
+'use strict'
+
+const tap = require('tap')
+const test = tap.test
+const fastURI = require('..')
+
+test('parse marks malformed authority and port inputs as errors', (t) => {
+  const malformedCases = [
+    {
+      input: 'http://[::1]foo',
+      expectedError: 'URI path must start with "/" when authority is present.'
+    },
+    {
+      input: 'http://[::1]:80abc/path',
+      expectedError: 'URI path must start with "/" when authority is present.'
+    },
+    {
+      input: 'http://example.com:80abc/path',
+      expectedError: 'URI path must start with "/" when authority is present.'
+    },
+    {
+      input: 'http://[::1]:65536',
+      expectedError: 'URI port is malformed.'
+    }
+  ]
+
+  t.plan(malformedCases.length)
+
+  malformedCases.forEach(({ input, expectedError }) => {
+    t.equal(fastURI.parse(input).error, expectedError, input)
+  })
+})
+
+test('normalize does not canonicalize malformed URLs into different valid URLs', (t) => {
+  const malformedCases = [
+    'http://[::1]foo',
+    'http://[::1]:80abc/path',
+    'http://example.com:80abc/path',
+    'http://[::1]:65536'
+  ]
+
+  t.plan(malformedCases.length)
+
+  malformedCases.forEach((input) => {
+    t.equal(fastURI.normalize(input), input, input)
+  })
+})
+
+test('equal returns false when either side is malformed', (t) => {
+  const malformedPairs = [
+    ['http://[::1]foo', 'http://[::1]/foo'],
+    ['http://[::1]:80abc/path', 'http://[::1]/abc/path'],
+    ['http://example.com:80abc/path', 'http://example.com/abc/path'],
+    ['http://[::1]:65536', 'http://[::1]:65536/']
+  ]
+
+  t.plan(malformedPairs.length)
+
+  malformedPairs.forEach(([left, right]) => {
+    t.equal(fastURI.equal(left, right), false, `${left} != ${right}`)
+  })
+})
+
+test('normalize preserves encoded authority delimiters in host', (t) => {
+  const cases = [
+    ['http://trusted.com%40evil.com/', 'http://trusted.com%40evil.com/'],
+    ['http://example.com%3A8080/', 'http://example.com%3A8080/'],
+    ['http://example.com%2Fevil.com/path', 'http://example.com%2Fevil.com/path'],
+    ['http://example.com%23fragment/path', 'http://example.com%23fragment/path'],
+    ['http://example.com%3Fq=evil/path', 'http://example.com%3Fq=evil/path'],
+    ['http://user%3Apass%40evil.com/', 'http://user%3Apass%40evil.com/'],
+    ['http://user@trusted.com%40evil.com/', 'http://user@trusted.com%40evil.com/'],
+    ['https://trusted.com%40evil.com/', 'https://trusted.com%40evil.com/'],
+    ['ws://trusted.com%40evil.com/chat', 'ws://trusted.com%40evil.com/chat'],
+    ['wss://trusted.com%40evil.com/chat', 'wss://trusted.com%40evil.com/chat']
+  ]
+
+  t.plan(cases.length)
+
+  cases.forEach(([input, expected]) => {
+    t.equal(fastURI.normalize(input), expected, input)
+  })
+})
+
+test('parse preserves encoded authority delimiters in host', (t) => {
+  const cases = [
+    ['http://trusted.com%40evil.com/', 'trusted.com%40evil.com'],
+    ['http://example.com%3A8080/', 'example.com%3A8080'],
+    ['http://user%3Apass%40evil.com/', 'user%3Apass%40evil.com']
+  ]
+
+  t.plan(cases.length)
+
+  cases.forEach(([input, expectedHost]) => {
+    t.equal(fastURI.parse(input).host, expectedHost, input)
+  })
+})
+
+test('equal returns false when encoded delimiters differ from live delimiters', (t) => {
+  const pairs = [
+    ['http://trusted.com%40evil.com/', 'http://trusted.com@evil.com/'],
+    ['http://example.com%3A8080/', 'http://example.com:8080/']
+  ]
+
+  t.plan(pairs.length)
+
+  pairs.forEach(([left, right]) => {
+    t.equal(fastURI.equal(left, right, {}), false, `${left} != ${right}`)
+  })
+})
+
+test('resolve preserves encoded authority delimiters', (t) => {
+  const result = fastURI.resolve('http://base.com/', '//trusted.com%40evil.com/path')
+  const parsed = fastURI.parse(result)
+
+  t.plan(1)
+  t.not(parsed.host, 'evil.com', '//trusted.com%40evil.com/path')
+})
+
+test('serialize escapes authority delimiters in host field', (t) => {
+  const result = fastURI.serialize({ scheme: 'http', host: 'trusted.com@evil.com', path: '/' })
+  const parsed = fastURI.parse(result)
+
+  t.plan(1)
+  t.not(parsed.host, 'evil.com', 'host: trusted.com@evil.com')
+})
+
+test('normalize does not double-decode %2540 into a live @', (t) => {
+  const result = fastURI.normalize('http://trusted.com%2540evil.com/')
+  const parsed = fastURI.parse(result)
+
+  t.plan(1)
+  t.not(parsed.host, 'trusted.com@evil.com', 'http://trusted.com%2540evil.com/')
+})