ires 0.1.3 → 0.1.4

data/ext/vendor/github.com/PuerkitoBio/purell/benchmarks/v0.1.0

@@ -0,0 +1,9 @@
+PASS
+BenchmarkSafe	  500000	      6131 ns/op
+BenchmarkUsuallySafe	  200000	      7864 ns/op
+BenchmarkUnsafe	  100000	     28560 ns/op
+BenchmarkAllDWORD	   50000	     38722 ns/op
+BenchmarkAllOctal	   50000	     40941 ns/op
+BenchmarkAllHex	   50000	     44063 ns/op
+BenchmarkAllCombined	   50000	     33613 ns/op
+ok  	github.com/PuerkitoBio/purell	17.404s

data/ext/vendor/github.com/PuerkitoBio/purell/example_test.go

@@ -0,0 +1,35 @@
+package purell
+
+import (
+	"fmt"
+	"net/url"
+)
+
+func ExampleNormalizeURLString() {
+	if normalized, err := NormalizeURLString("hTTp://someWEBsite.com:80/Amazing%3f/url/",
+		FlagLowercaseScheme|FlagLowercaseHost|FlagUppercaseEscapes); err != nil {
+		panic(err)
+	} else {
+		fmt.Print(normalized)
+	}
+	// Output: http://somewebsite.com:80/Amazing%3F/url/
+}
+
+func ExampleMustNormalizeURLString() {
+	normalized := MustNormalizeURLString("hTTpS://someWEBsite.com:443/Amazing%fa/url/",
+		FlagsUnsafeGreedy)
+	fmt.Print(normalized)
+
+	// Output: http://somewebsite.com/Amazing%FA/url
+}
+
+func ExampleNormalizeURL() {
+	if u, err := url.Parse("Http://SomeUrl.com:8080/a/b/.././c///g?c=3&a=1&b=9&c=0#target"); err != nil {
+		panic(err)
+	} else {
+		normalized := NormalizeURL(u, FlagsUsuallySafeGreedy|FlagRemoveDuplicateSlashes|FlagRemoveFragment)
+		fmt.Print(normalized)
+	}
+
+	// Output: http://someurl.com:8080/a/c/g?c=3&a=1&b=9&c=0
+}

data/ext/vendor/github.com/PuerkitoBio/purell/purell.go

@@ -0,0 +1,379 @@
+/*
+Package purell offers URL normalization as described on the wikipedia page:
+http://en.wikipedia.org/wiki/URL_normalization
+*/
+package purell
+
+import (
+	"bytes"
+	"fmt"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/urlesc"
+	"golang.org/x/net/idna"
+	"golang.org/x/text/unicode/norm"
+	"golang.org/x/text/width"
+)
+
+// A set of normalization flags determines how a URL will
+// be normalized.
+type NormalizationFlags uint
+
+const (
+	// Safe normalizations
+	FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
+	FlagLowercaseHost                                            // http://HOST -> http://host
+	FlagUppercaseEscapes                                         // http://host/t%ef -> http://host/t%EF
+	FlagDecodeUnnecessaryEscapes                                 // http://host/t%41 -> http://host/tA
+	FlagEncodeNecessaryEscapes                                   // http://host/!"#$ -> http://host/%21%22#$
+	FlagRemoveDefaultPort                                        // http://host:80 -> http://host
+	FlagRemoveEmptyQuerySeparator                                // http://host/path? -> http://host/path
+
+	// Usually safe normalizations
+	FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
+	FlagAddTrailingSlash    // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
+	FlagRemoveDotSegments   // http://host/path/./a/b/../c -> http://host/path/a/c
+
+	// Unsafe normalizations
+	FlagRemoveDirectoryIndex   // http://host/path/index.html -> http://host/path/
+	FlagRemoveFragment         // http://host/path#fragment -> http://host/path
+	FlagForceHTTP              // https://host -> http://host
+	FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
+	FlagRemoveWWW              // http://www.host/ -> http://host/
+	FlagAddWWW                 // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
+	FlagSortQuery              // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
+
+	// Normalizations not in the wikipedia article, required to cover tests cases
+	// submitted by jehiah
+	FlagDecodeDWORDHost           // http://1113982867 -> http://66.102.7.147
+	FlagDecodeOctalHost           // http://0102.0146.07.0223 -> http://66.102.7.147
+	FlagDecodeHexHost             // http://0x42660793 -> http://66.102.7.147
+	FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
+	FlagRemoveEmptyPortSeparator  // http://host:/path -> http://host/path
+
+	// Convenience set of safe normalizations
+	FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
+
+	// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
+	// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
+
+	// Convenience set of usually safe normalizations (includes FlagsSafe)
+	FlagsUsuallySafeGreedy    NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
+	FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
+
+	// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
+	FlagsUnsafeGreedy    NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
+	FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
+
+	// Convenience set of all available flags
+	FlagsAllGreedy    = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
+	FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
+)
+
+const (
+	defaultHttpPort  = ":80"
+	defaultHttpsPort = ":443"
+)
+
+// Regular expressions used by the normalizations
+var rxPort = regexp.MustCompile(`(:\d+)/?$`)
+var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`)
+var rxDupSlashes = regexp.MustCompile(`/{2,}`)
+var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`)
+var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`)
+var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`)
+var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`)
+var rxEmptyPort = regexp.MustCompile(`:+$`)
+
+// Map of flags to implementation function.
+// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically
+// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator.
+
+// Since maps have undefined traversing order, make a slice of ordered keys
+var flagsOrder = []NormalizationFlags{
+	FlagLowercaseScheme,
+	FlagLowercaseHost,
+	FlagRemoveDefaultPort,
+	FlagRemoveDirectoryIndex,
+	FlagRemoveDotSegments,
+	FlagRemoveFragment,
+	FlagForceHTTP, // Must be after remove default port (because https=443/http=80)
+	FlagRemoveDuplicateSlashes,
+	FlagRemoveWWW,
+	FlagAddWWW,
+	FlagSortQuery,
+	FlagDecodeDWORDHost,
+	FlagDecodeOctalHost,
+	FlagDecodeHexHost,
+	FlagRemoveUnnecessaryHostDots,
+	FlagRemoveEmptyPortSeparator,
+	FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last
+	FlagAddTrailingSlash,
+}
+
+// ... and then the map, where order is unimportant
+var flags = map[NormalizationFlags]func(*url.URL){
+	FlagLowercaseScheme:           lowercaseScheme,
+	FlagLowercaseHost:             lowercaseHost,
+	FlagRemoveDefaultPort:         removeDefaultPort,
+	FlagRemoveDirectoryIndex:      removeDirectoryIndex,
+	FlagRemoveDotSegments:         removeDotSegments,
+	FlagRemoveFragment:            removeFragment,
+	FlagForceHTTP:                 forceHTTP,
+	FlagRemoveDuplicateSlashes:    removeDuplicateSlashes,
+	FlagRemoveWWW:                 removeWWW,
+	FlagAddWWW:                    addWWW,
+	FlagSortQuery:                 sortQuery,
+	FlagDecodeDWORDHost:           decodeDWORDHost,
+	FlagDecodeOctalHost:           decodeOctalHost,
+	FlagDecodeHexHost:             decodeHexHost,
+	FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots,
+	FlagRemoveEmptyPortSeparator:  removeEmptyPortSeparator,
+	FlagRemoveTrailingSlash:       removeTrailingSlash,
+	FlagAddTrailingSlash:          addTrailingSlash,
+}
+
+// MustNormalizeURLString returns the normalized string, and panics if an error occurs.
+// It takes an URL string as input, as well as the normalization flags.
+func MustNormalizeURLString(u string, f NormalizationFlags) string {
+	result, e := NormalizeURLString(u, f)
+	if e != nil {
+		panic(e)
+	}
+	return result
+}
+
+// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
+// It takes an URL string as input, as well as the normalization flags.
+func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
+	parsed, err := url.Parse(u)
+	if err != nil {
+		return "", err
+	}
+
+	if f&FlagLowercaseHost == FlagLowercaseHost {
+		parsed.Host = strings.ToLower(parsed.Host)
+	}
+
+	// The idna package doesn't fully conform to RFC 5895
+	// (https://tools.ietf.org/html/rfc5895), so we do it here.
+	// Taken from Go 1.8 cycle source, courtesy of bradfitz.
+	// TODO: Remove when (if?) idna package conforms to RFC 5895.
+	parsed.Host = width.Fold.String(parsed.Host)
+	parsed.Host = norm.NFC.String(parsed.Host)
+	if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil {
+		return "", err
+	}
+
+	return NormalizeURL(parsed, f), nil
+}
+
+// NormalizeURL returns the normalized string.
+// It takes a parsed URL object as input, as well as the normalization flags.
+func NormalizeURL(u *url.URL, f NormalizationFlags) string {
+	for _, k := range flagsOrder {
+		if f&k == k {
+			flags[k](u)
+		}
+	}
+	return urlesc.Escape(u)
+}
+
+func lowercaseScheme(u *url.URL) {
+	if len(u.Scheme) > 0 {
+		u.Scheme = strings.ToLower(u.Scheme)
+	}
+}
+
+func lowercaseHost(u *url.URL) {
+	if len(u.Host) > 0 {
+		u.Host = strings.ToLower(u.Host)
+	}
+}
+
+func removeDefaultPort(u *url.URL) {
+	if len(u.Host) > 0 {
+		scheme := strings.ToLower(u.Scheme)
+		u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string {
+			if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) {
+				return ""
+			}
+			return val
+		})
+	}
+}
+
+func removeTrailingSlash(u *url.URL) {
+	if l := len(u.Path); l > 0 {
+		if strings.HasSuffix(u.Path, "/") {
+			u.Path = u.Path[:l-1]
+		}
+	} else if l = len(u.Host); l > 0 {
+		if strings.HasSuffix(u.Host, "/") {
+			u.Host = u.Host[:l-1]
+		}
+	}
+}
+
+func addTrailingSlash(u *url.URL) {
+	if l := len(u.Path); l > 0 {
+		if !strings.HasSuffix(u.Path, "/") {
+			u.Path += "/"
+		}
+	} else if l = len(u.Host); l > 0 {
+		if !strings.HasSuffix(u.Host, "/") {
+			u.Host += "/"
+		}
+	}
+}
+
+func removeDotSegments(u *url.URL) {
+	if len(u.Path) > 0 {
+		var dotFree []string
+		var lastIsDot bool
+
+		sections := strings.Split(u.Path, "/")
+		for _, s := range sections {
+			if s == ".." {
+				if len(dotFree) > 0 {
+					dotFree = dotFree[:len(dotFree)-1]
+				}
+			} else if s != "." {
+				dotFree = append(dotFree, s)
+			}
+			lastIsDot = (s == "." || s == "..")
+		}
+		// Special case if host does not end with / and new path does not begin with /
+		u.Path = strings.Join(dotFree, "/")
+		if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") {
+			u.Path = "/" + u.Path
+		}
+		// Special case if the last segment was a dot, make sure the path ends with a slash
+		if lastIsDot && !strings.HasSuffix(u.Path, "/") {
+			u.Path += "/"
+		}
+	}
+}
+
+func removeDirectoryIndex(u *url.URL) {
+	if len(u.Path) > 0 {
+		u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1")
+	}
+}
+
+func removeFragment(u *url.URL) {
+	u.Fragment = ""
+}
+
+func forceHTTP(u *url.URL) {
+	if strings.ToLower(u.Scheme) == "https" {
+		u.Scheme = "http"
+	}
+}
+
+func removeDuplicateSlashes(u *url.URL) {
+	if len(u.Path) > 0 {
+		u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/")
+	}
+}
+
+func removeWWW(u *url.URL) {
+	if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") {
+		u.Host = u.Host[4:]
+	}
+}
+
+func addWWW(u *url.URL) {
+	if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") {
+		u.Host = "www." + u.Host
+	}
+}
+
+func sortQuery(u *url.URL) {
+	q := u.Query()
+
+	if len(q) > 0 {
+		arKeys := make([]string, len(q))
+		i := 0
+		for k, _ := range q {
+			arKeys[i] = k
+			i++
+		}
+		sort.Strings(arKeys)
+		buf := new(bytes.Buffer)
+		for _, k := range arKeys {
+			sort.Strings(q[k])
+			for _, v := range q[k] {
+				if buf.Len() > 0 {
+					buf.WriteRune('&')
+				}
+				buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v)))
+			}
+		}
+
+		// Rebuild the raw query string
+		u.RawQuery = buf.String()
+	}
+}
+
+func decodeDWORDHost(u *url.URL) {
+	if len(u.Host) > 0 {
+		if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 {
+			var parts [4]int64
+
+			dword, _ := strconv.ParseInt(matches[1], 10, 0)
+			for i, shift := range []uint{24, 16, 8, 0} {
+				parts[i] = dword >> shift & 0xFF
+			}
+			u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2])
+		}
+	}
+}
+
+func decodeOctalHost(u *url.URL) {
+	if len(u.Host) > 0 {
+		if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 {
+			var parts [4]int64
+
+			for i := 1; i <= 4; i++ {
+				parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0)
+			}
+			u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5])
+		}
+	}
+}
+
+func decodeHexHost(u *url.URL) {
+	if len(u.Host) > 0 {
+		if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 {
+			// Conversion is safe because of regex validation
+			parsed, _ := strconv.ParseInt(matches[1], 16, 0)
+			// Set host as DWORD (base 10) encoded host
+			u.Host = fmt.Sprintf("%d%s", parsed, matches[2])
+			// The rest is the same as decoding a DWORD host
+			decodeDWORDHost(u)
+		}
+	}
+}
+
+func removeUnncessaryHostDots(u *url.URL) {
+	if len(u.Host) > 0 {
+		if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 {
+			// Trim the leading and trailing dots
+			u.Host = strings.Trim(matches[1], ".")
+			if len(matches) > 2 {
+				u.Host += matches[2]
+			}
+		}
+	}
+}
+
+func removeEmptyPortSeparator(u *url.URL) {
+	if len(u.Host) > 0 {
+		u.Host = rxEmptyPort.ReplaceAllString(u.Host, "")
+	}
+}

data/ext/vendor/github.com/PuerkitoBio/purell/purell_test.go

@@ -0,0 +1,768 @@
+package purell
+
+import (
+	"fmt"
+	"net/url"
+	"testing"
+)
+
+type testCase struct {
+	nm     string
+	src    string
+	flgs   NormalizationFlags
+	res    string
+	parsed bool
+}
+
+var (
+	cases = [...]*testCase{
+		&testCase{
+			"LowerScheme",
+			"HTTP://www.SRC.ca",
+			FlagLowercaseScheme,
+			"http://www.SRC.ca",
+			false,
+		},
+		&testCase{
+			"LowerScheme2",
+			"http://www.SRC.ca",
+			FlagLowercaseScheme,
+			"http://www.SRC.ca",
+			false,
+		},
+		&testCase{
+			"LowerHost",
+			"HTTP://www.SRC.ca/",
+			FlagLowercaseHost,
+			"http://www.src.ca/", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"UpperEscapes",
+			`http://www.whatever.com/Some%aa%20Special%8Ecases/`,
+			FlagUppercaseEscapes,
+			"http://www.whatever.com/Some%AA%20Special%8Ecases/",
+			false,
+		},
+		&testCase{
+			"UnnecessaryEscapes",
+			`http://www.toto.com/%41%42%2E%44/%32%33%52%2D/%5f%7E`,
+			FlagDecodeUnnecessaryEscapes,
+			"http://www.toto.com/AB.D/23R-/_~",
+			false,
+		},
+		&testCase{
+			"RemoveDefaultPort",
+			"HTTP://www.SRC.ca:80/",
+			FlagRemoveDefaultPort,
+			"http://www.SRC.ca/", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"RemoveDefaultPort2",
+			"HTTP://www.SRC.ca:80",
+			FlagRemoveDefaultPort,
+			"http://www.SRC.ca", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"RemoveDefaultPort3",
+			"HTTP://www.SRC.ca:8080",
+			FlagRemoveDefaultPort,
+			"http://www.SRC.ca:8080", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"Safe",
+			"HTTP://www.SRC.ca:80/to%1ato%8b%ee/OKnow%41%42%43%7e",
+			FlagsSafe,
+			"http://www.src.ca/to%1Ato%8B%EE/OKnowABC~",
+			false,
+		},
+		&testCase{
+			"BothLower",
+			"HTTP://www.SRC.ca:80/to%1ato%8b%ee/OKnow%41%42%43%7e",
+			FlagLowercaseHost | FlagLowercaseScheme,
+			"http://www.src.ca:80/to%1Ato%8B%EE/OKnowABC~",
+			false,
+		},
+		&testCase{
+			"RemoveTrailingSlash",
+			"HTTP://www.SRC.ca:80/",
+			FlagRemoveTrailingSlash,
+			"http://www.SRC.ca:80", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"RemoveTrailingSlash2",
+			"HTTP://www.SRC.ca:80/toto/titi/",
+			FlagRemoveTrailingSlash,
+			"http://www.SRC.ca:80/toto/titi", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"RemoveTrailingSlash3",
+			"HTTP://www.SRC.ca:80/toto/titi/fin/?a=1",
+			FlagRemoveTrailingSlash,
+			"http://www.SRC.ca:80/toto/titi/fin?a=1", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"AddTrailingSlash",
+			"HTTP://www.SRC.ca:80",
+			FlagAddTrailingSlash,
+			"http://www.SRC.ca:80/", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"AddTrailingSlash2",
+			"HTTP://www.SRC.ca:80/toto/titi.html",
+			FlagAddTrailingSlash,
+			"http://www.SRC.ca:80/toto/titi.html/", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"AddTrailingSlash3",
+			"HTTP://www.SRC.ca:80/toto/titi/fin?a=1",
+			FlagAddTrailingSlash,
+			"http://www.SRC.ca:80/toto/titi/fin/?a=1", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"RemoveDotSegments",
+			"HTTP://root/a/b/./../../c/",
+			FlagRemoveDotSegments,
+			"http://root/c/", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"RemoveDotSegments2",
+			"HTTP://root/../a/b/./../c/../d",
+			FlagRemoveDotSegments,
+			"http://root/a/d", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"UsuallySafe",
+			"HTTP://www.SRC.ca:80/to%1ato%8b%ee/./c/d/../OKnow%41%42%43%7e/?a=b#test",
+			FlagsUsuallySafeGreedy,
+			"http://www.src.ca/to%1Ato%8B%EE/c/OKnowABC~?a=b#test",
+			false,
+		},
+		&testCase{
+			"RemoveDirectoryIndex",
+			"HTTP://root/a/b/c/default.aspx",
+			FlagRemoveDirectoryIndex,
+			"http://root/a/b/c/", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"RemoveDirectoryIndex2",
+			"HTTP://root/a/b/c/default#a=b",
+			FlagRemoveDirectoryIndex,
+			"http://root/a/b/c/default#a=b", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"RemoveFragment",
+			"HTTP://root/a/b/c/default#toto=tata",
+			FlagRemoveFragment,
+			"http://root/a/b/c/default", // Since Go1.1, scheme is automatically lowercased
+			false,
+		},
+		&testCase{
+			"ForceHTTP",
+			"https://root/a/b/c/default#toto=tata",
+			FlagForceHTTP,
+			"http://root/a/b/c/default#toto=tata",
+			false,
+		},
+		&testCase{
+			"RemoveDuplicateSlashes",
+			"https://root/a//b///c////default#toto=tata",
+			FlagRemoveDuplicateSlashes,
+			"https://root/a/b/c/default#toto=tata",
+			false,
+		},
+		&testCase{
+			"RemoveDuplicateSlashes2",
+			"https://root//a//b///c////default#toto=tata",
+			FlagRemoveDuplicateSlashes,
+			"https://root/a/b/c/default#toto=tata",
+			false,
+		},
+		&testCase{
+			"RemoveWWW",
+			"https://www.root/a/b/c/",
+			FlagRemoveWWW,
+			"https://root/a/b/c/",
+			false,
+		},
+		&testCase{
+			"RemoveWWW2",
+			"https://WwW.Root/a/b/c/",
+			FlagRemoveWWW,
+			"https://Root/a/b/c/",
+			false,
+		},
+		&testCase{
+			"AddWWW",
+			"https://Root/a/b/c/",
+			FlagAddWWW,
+			"https://www.Root/a/b/c/",
+			false,
+		},
+		&testCase{
+			"SortQuery",
+			"http://root/toto/?b=4&a=1&c=3&b=2&a=5",
+			FlagSortQuery,
+			"http://root/toto/?a=1&a=5&b=2&b=4&c=3",
+			false,
+		},
+		&testCase{
+			"RemoveEmptyQuerySeparator",
+			"http://root/toto/?",
+			FlagRemoveEmptyQuerySeparator,
+			"http://root/toto/",
+			false,
+		},
+		&testCase{
+			"Unsafe",
+			"HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+			FlagsUnsafeGreedy,
+			"http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3",
+			false,
+		},
+		&testCase{
+			"Safe2",
+			"HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+			FlagsSafe,
+			"https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+			false,
+		},
+		&testCase{
+			"UsuallySafe2",
+			"HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+			FlagsUsuallySafeGreedy,
+			"https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid",
+			false,
+		},
+		&testCase{
+			"AddTrailingSlashBug",
+			"http://src.ca/",
+			FlagsAllNonGreedy,
+			"http://www.src.ca/",
+			false,
+		},
+		&testCase{
+			"SourceModified",
+			"HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid",
+			FlagsUnsafeGreedy,
+			"http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3",
+			true,
+		},
+		&testCase{
+			"IPv6-1",
+			"http://[2001:db8:1f70::999:de8:7648:6e8]/test",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://[2001:db8:1f70::999:de8:7648:6e8]/test",
+			false,
+		},
+		&testCase{
+			"IPv6-2",
+			"http://[::ffff:192.168.1.1]/test",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://[::ffff:192.168.1.1]/test",
+			false,
+		},
+		&testCase{
+			"IPv6-3",
+			"http://[::ffff:192.168.1.1]:80/test",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://[::ffff:192.168.1.1]/test",
+			false,
+		},
+		&testCase{
+			"IPv6-4",
+			"htTps://[::fFff:192.168.1.1]:443/test",
+			FlagsSafe | FlagRemoveDotSegments,
+			"https://[::ffff:192.168.1.1]/test",
+			false,
+		},
+		&testCase{
+			"FTP",
+			"ftp://user:pass@ftp.foo.net/foo/bar",
+			FlagsSafe | FlagRemoveDotSegments,
+			"ftp://user:pass@ftp.foo.net/foo/bar",
+			false,
+		},
+		&testCase{
+			"Standard-1",
+			"http://www.foo.com:80/foo",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://www.foo.com/foo",
+			false,
+		},
+		&testCase{
+			"Standard-2",
+			"http://www.foo.com:8000/foo",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://www.foo.com:8000/foo",
+			false,
+		},
+		&testCase{
+			"Standard-3",
+			"http://www.foo.com/%7ebar",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://www.foo.com/~bar",
+			false,
+		},
+		&testCase{
+			"Standard-4",
+			"http://www.foo.com/%7Ebar",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://www.foo.com/~bar",
+			false,
+		},
+		&testCase{
+			"Standard-5",
+			"http://USER:pass@www.Example.COM/foo/bar",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://USER:pass@www.example.com/foo/bar",
+			false,
+		},
+		&testCase{
+			"Standard-6",
+			"http://test.example/?a=%26&b=1",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://test.example/?a=%26&b=1",
+			false,
+		},
+		&testCase{
+			"Standard-7",
+			"http://test.example/%25/?p=%20val%20%25",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://test.example/%25/?p=%20val%20%25",
+			false,
+		},
+		&testCase{
+			"Standard-8",
+			"http://test.example/path/with a%20space+/",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://test.example/path/with%20a%20space+/",
+			false,
+		},
+		&testCase{
+			"Standard-9",
+			"http://test.example/?",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://test.example/",
+			false,
+		},
+		&testCase{
+			"Standard-10",
+			"http://a.COM/path/?b&a",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://a.com/path/?b&a",
+			false,
+		},
+		&testCase{
+			"StandardCasesAddTrailingSlash",
+			"http://test.example?",
+			FlagsSafe | FlagAddTrailingSlash,
+			"http://test.example/",
+			false,
+		},
+		&testCase{
+			"OctalIP-1",
+			"http://0123.011.0.4/",
+			FlagsSafe | FlagDecodeOctalHost,
+			"http://0123.011.0.4/",
+			false,
+		},
+		&testCase{
+			"OctalIP-2",
+			"http://0102.0146.07.0223/",
+			FlagsSafe | FlagDecodeOctalHost,
+			"http://66.102.7.147/",
+			false,
+		},
+		&testCase{
+			"OctalIP-3",
+			"http://0102.0146.07.0223.:23/",
+			FlagsSafe | FlagDecodeOctalHost,
+			"http://66.102.7.147.:23/",
+			false,
+		},
+		&testCase{
+			"OctalIP-4",
+			"http://USER:pass@0102.0146.07.0223../",
+			FlagsSafe | FlagDecodeOctalHost,
+			"http://USER:pass@66.102.7.147../",
+			false,
+		},
+		&testCase{
+			"DWORDIP-1",
+			"http://123.1113982867/",
+			FlagsSafe | FlagDecodeDWORDHost,
+			"http://123.1113982867/",
+			false,
+		},
+		&testCase{
+			"DWORDIP-2",
+			"http://1113982867/",
+			FlagsSafe | FlagDecodeDWORDHost,
+			"http://66.102.7.147/",
+			false,
+		},
+		&testCase{
+			"DWORDIP-3",
+			"http://1113982867.:23/",
+			FlagsSafe | FlagDecodeDWORDHost,
+			"http://66.102.7.147.:23/",
+			false,
+		},
+		&testCase{
+			"DWORDIP-4",
+			"http://USER:pass@1113982867../",
+			FlagsSafe | FlagDecodeDWORDHost,
+			"http://USER:pass@66.102.7.147../",
+			false,
+		},
+		&testCase{
+			"HexIP-1",
+			"http://0x123.1113982867/",
+			FlagsSafe | FlagDecodeHexHost,
+			"http://0x123.1113982867/",
+			false,
+		},
+		&testCase{
+			"HexIP-2",
+			"http://0x42660793/",
+			FlagsSafe | FlagDecodeHexHost,
+			"http://66.102.7.147/",
+			false,
+		},
+		&testCase{
+			"HexIP-3",
+			"http://0x42660793.:23/",
+			FlagsSafe | FlagDecodeHexHost,
+			"http://66.102.7.147.:23/",
+			false,
+		},
+		&testCase{
+			"HexIP-4",
+			"http://USER:pass@0x42660793../",
+			FlagsSafe | FlagDecodeHexHost,
+			"http://USER:pass@66.102.7.147../",
+			false,
+		},
+		&testCase{
+			"UnnecessaryHostDots-1",
+			"http://.www.foo.com../foo/bar.html",
+			FlagsSafe | FlagRemoveUnnecessaryHostDots,
+			"http://www.foo.com/foo/bar.html",
+			false,
+		},
+		&testCase{
+			"UnnecessaryHostDots-2",
+			"http://www.foo.com./foo/bar.html",
+			FlagsSafe | FlagRemoveUnnecessaryHostDots,
+			"http://www.foo.com/foo/bar.html",
+			false,
+		},
+		&testCase{
+			"UnnecessaryHostDots-3",
+			"http://www.foo.com.:81/foo",
+			FlagsSafe | FlagRemoveUnnecessaryHostDots,
+			"http://www.foo.com:81/foo",
+			false,
+		},
+		&testCase{
+			"UnnecessaryHostDots-4",
+			"http://www.example.com./",
+			FlagsSafe | FlagRemoveUnnecessaryHostDots,
+			"http://www.example.com/",
+			false,
+		},
+		&testCase{
+			"EmptyPort-1",
+			"http://www.thedraymin.co.uk:/main/?p=308",
+			FlagsSafe | FlagRemoveEmptyPortSeparator,
+			"http://www.thedraymin.co.uk/main/?p=308",
+			false,
+		},
+		&testCase{
+			"EmptyPort-2",
+			"http://www.src.ca:",
+			FlagsSafe | FlagRemoveEmptyPortSeparator,
+			"http://www.src.ca",
+			false,
+		},
+		&testCase{
+			"Slashes-1",
+			"http://test.example/foo/bar/.",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/bar/",
+			false,
+		},
+		&testCase{
+			"Slashes-2",
+			"http://test.example/foo/bar/./",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/bar/",
+			false,
+		},
+		&testCase{
+			"Slashes-3",
+			"http://test.example/foo/bar/..",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/",
+			false,
+		},
+		&testCase{
+			"Slashes-4",
+			"http://test.example/foo/bar/../",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/",
+			false,
+		},
+		&testCase{
+			"Slashes-5",
+			"http://test.example/foo/bar/../baz",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/baz",
+			false,
+		},
+		&testCase{
+			"Slashes-6",
+			"http://test.example/foo/bar/../..",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/",
+			false,
+		},
+		&testCase{
+			"Slashes-7",
+			"http://test.example/foo/bar/../../",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/",
+			false,
+		},
+		&testCase{
+			"Slashes-8",
+			"http://test.example/foo/bar/../../baz",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/baz",
+			false,
+		},
+		&testCase{
+			"Slashes-9",
+			"http://test.example/foo/bar/../../../baz",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/baz",
+			false,
+		},
+		&testCase{
+			"Slashes-10",
+			"http://test.example/foo/bar/../../../../baz",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/baz",
+			false,
+		},
+		&testCase{
+			"Slashes-11",
+			"http://test.example/./foo",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo",
+			false,
+		},
+		&testCase{
+			"Slashes-12",
+			"http://test.example/../foo",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo",
+			false,
+		},
+		&testCase{
+			"Slashes-13",
+			"http://test.example/foo.",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo.",
+			false,
+		},
+		&testCase{
+			"Slashes-14",
+			"http://test.example/.foo",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/.foo",
+			false,
+		},
+		&testCase{
+			"Slashes-15",
+			"http://test.example/foo..",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo..",
+			false,
+		},
+		&testCase{
+			"Slashes-16",
+			"http://test.example/..foo",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/..foo",
+			false,
+		},
+		&testCase{
+			"Slashes-17",
+			"http://test.example/./../foo",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo",
+			false,
+		},
+		&testCase{
+			"Slashes-18",
+			"http://test.example/./foo/.",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/",
+			false,
+		},
+		&testCase{
+			"Slashes-19",
+			"http://test.example/foo/./bar",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/bar",
+			false,
+		},
+		&testCase{
+			"Slashes-20",
+			"http://test.example/foo/../bar",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/bar",
+			false,
+		},
+		&testCase{
+			"Slashes-21",
+			"http://test.example/foo//",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/",
+			false,
+		},
+		&testCase{
+			"Slashes-22",
+			"http://test.example/foo///bar//",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"http://test.example/foo/bar/",
+			false,
+		},
+		&testCase{
+			"Relative",
+			"foo/bar",
+			FlagsAllGreedy,
+			"foo/bar",
+			false,
+		},
+		&testCase{
+			"Relative-1",
+			"./../foo",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"foo",
+			false,
+		},
+		&testCase{
+			"Relative-2",
+			"./foo/bar/../baz/../bang/..",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"foo/",
+			false,
+		},
+		&testCase{
+			"Relative-3",
+			"foo///bar//",
+			FlagsSafe | FlagRemoveDotSegments | FlagRemoveDuplicateSlashes,
+			"foo/bar/",
+			false,
+		},
+		&testCase{
+			"Relative-4",
+			"www.youtube.com",
+			FlagsUsuallySafeGreedy,
+			"www.youtube.com",
+			false,
+		},
+		/*&testCase{
+			"UrlNorm-5",
+			"http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3",
+			FlagsSafe | FlagRemoveDotSegments,
+			"http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3",
+			false,
+		},
+		&testCase{
+			"UrlNorm-1",
+			"http://test.example/?a=%e3%82%82%26",
+			FlagsAllGreedy,
+			"http://test.example/?a=\xe3\x82\x82%26",
+			false,
+		},*/
+	}
+)
+
+func TestRunner(t *testing.T) {
+	for _, tc := range cases {
+		runCase(tc, t)
+	}
+}
+
+func runCase(tc *testCase, t *testing.T) {
+	t.Logf("running %s...", tc.nm)
+	if tc.parsed {
+		u, e := url.Parse(tc.src)
+		if e != nil {
+			t.Errorf("%s - FAIL : %s", tc.nm, e)
+			return
+		} else {
+			NormalizeURL(u, tc.flgs)
+			if s := u.String(); s != tc.res {
+				t.Errorf("%s - FAIL expected '%s', got '%s'", tc.nm, tc.res, s)
+			}
+		}
+	} else {
+		if s, e := NormalizeURLString(tc.src, tc.flgs); e != nil {
+			t.Errorf("%s - FAIL : %s", tc.nm, e)
+		} else if s != tc.res {
+			t.Errorf("%s - FAIL expected '%s', got '%s'", tc.nm, tc.res, s)
+		}
+	}
+}
+
+func TestDecodeUnnecessaryEscapesAll(t *testing.T) {
+	var url = "http://host/"
+
+	for i := 0; i < 256; i++ {
+		url += fmt.Sprintf("%%%02x", i)
+	}
+	if s, e := NormalizeURLString(url, FlagDecodeUnnecessaryEscapes); e != nil {
+		t.Fatalf("Got error %s", e.Error())
+	} else {
+		const want = "http://host/%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25&'()*+,-./0123456789:;%3C=%3E%3F@ABCDEFGHIJKLMNOPQRSTUVWXYZ[%5C]%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF"
+		if s != want {
+			t.Errorf("DecodeUnnecessaryEscapesAll:\nwant\n%s\ngot\n%s", want, s)
+		}
+	}
+}
+
+func TestEncodeNecessaryEscapesAll(t *testing.T) {
+	var url = "http://host/"
+
+	for i := 0; i < 256; i++ {
+		if i != 0x25 {
+			url += string(i)
+		}
+	}
+	if s, e := NormalizeURLString(url, FlagEncodeNecessaryEscapes); e != nil {
+		t.Fatalf("Got error %s", e.Error())
+	} else {
+		const want = "http://host/%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22#$&'()*+,-./0123456789:;%3C=%3E?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[%5C]%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%C2%80%C2%81%C2%82%C2%83%C2%84%C2%85%C2%86%C2%87%C2%88%C2%89%C2%8A%C2%8B%C2%8C%C2%8D%C2%8E%C2%8F%C2%90%C2%91%C2%92%C2%93%C2%94%C2%95%C2%96%C2%97%C2%98%C2%99%C2%9A%C2%9B%C2%9C%C2%9D%C2%9E%C2%9F%C2%A0%C2%A1%C2%A2%C2%A3%C2%A4%C2%A5%C2%A6%C2%A7%C2%A8%C2%A9%C2%AA%C2%AB%C2%AC%C2%AD%C2%AE%C2%AF%C2%B0%C2%B1%C2%B2%C2%B3%C2%B4%C2%B5%C2%B6%C2%B7%C2%B8%C2%B9%C2%BA%C2%BB%C2%BC%C2%BD%C2%BE%C2%BF%C3%80%C3%81%C3%82%C3%83%C3%84%C3%85%C3%86%C3%87%C3%88%C3%89%C3%8A%C3%8B%C3%8C%C3%8D%C3%8E%C3%8F%C3%90%C3%91%C3%92%C3%93%C3%94%C3%95%C3%96%C3%97%C3%98%C3%99%C3%9A%C3%9B%C3%9C%C3%9D%C3%9E%C3%9F%C3%A0%C3%A1%C3%A2%C3%A3%C3%A4%C3%A5%C3%A6%C3%A7%C3%A8%C3%A9%C3%AA%C3%AB%C3%AC%C3%AD%C3%AE%C3%AF%C3%B0%C3%B1%C3%B2%C3%B3%C3%B4%C3%B5%C3%B6%C3%B7%C3%B8%C3%B9%C3%BA%C3%BB%C3%BC%C3%BD%C3%BE%C3%BF"
+		if s != want {
+			t.Errorf("EncodeNecessaryEscapesAll:\nwant\n%s\ngot\n%s", want, s)
+		}
+	}
+}