ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/cases/gen_trieval.go

@@ -0,0 +1,219 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+// This file contains definitions for interpreting the trie value of the case
+// trie generated by "go run gen*.go". It is shared by both the generator
+// program and the resultant package. Sharing is achieved by the generator
+// copying gen_trieval.go to trieval.go and changing what's above this comment.
+
+// info holds case information for a single rune. It is the value returned
+// by a trie lookup. Most mapping information can be stored in a single 16-bit
+// value. If not, for example when a rune is mapped to multiple runes, the value
+// stores some basic case data and an index into an array with additional data.
+//
+// The per-rune values have the following format:
+//
+//   if (exception) {
+//     15..5  unsigned exception index
+//         4  unused
+//   } else {
+//     15..8  XOR pattern or index to XOR pattern for case mapping
+//            Only 13..8 are used for XOR patterns.
+//         7  inverseFold (fold to upper, not to lower)
+//         6  index: interpret the XOR pattern as an index
+//            or isMid if case mode is cIgnorableUncased.
+//      5..4  CCC: zero (normal or break), above or other
+//   }
+//      3  exception: interpret this value as an exception index
+//         (TODO: is this bit necessary? Probably implied from case mode.)
+//   2..0  case mode
+//
+// For the non-exceptional cases, a rune must be either uncased, lowercase or
+// uppercase. If the rune is cased, the XOR pattern maps either a lowercase
+// rune to uppercase or an uppercase rune to lowercase (applied to the 10
+// least-significant bits of the rune).
+//
+// See the definitions below for a more detailed description of the various
+// bits.
+type info uint16
+
+const (
+	casedMask      = 0x0003
+	fullCasedMask  = 0x0007
+	ignorableMask  = 0x0006
+	ignorableValue = 0x0004
+
+	inverseFoldBit = 1 << 7
+	isMidBit       = 1 << 6
+
+	exceptionBit     = 1 << 3
+	exceptionShift   = 5
+	numExceptionBits = 11
+
+	xorIndexBit = 1 << 6
+	xorShift    = 8
+
+	// There is no mapping if all xor bits and the exception bit are zero.
+	hasMappingMask = 0xff80 | exceptionBit
+)
+
+// The case mode bits encodes the case type of a rune. This includes uncased,
+// title, upper and lower case and case ignorable. (For a definition of these
+// terms see Chapter 3 of The Unicode Standard Core Specification.) In some rare
+// cases, a rune can be both cased and case-ignorable. This is encoded by
+// cIgnorableCased. A rune of this type is always lower case. Some runes are
+// cased while not having a mapping.
+//
+// A common pattern for scripts in the Unicode standard is for upper and lower
+// case runes to alternate for increasing rune values (e.g. the accented Latin
+// ranges starting from U+0100 and U+1E00 among others and some Cyrillic
+// characters). We use this property by defining a cXORCase mode, where the case
+// mode (always upper or lower case) is derived from the rune value. As the XOR
+// pattern for case mappings is often identical for successive runes, using
+// cXORCase can result in large series of identical trie values. This, in turn,
+// allows us to better compress the trie blocks.
+const (
+	cUncased          info = iota // 000
+	cTitle                        // 001
+	cLower                        // 010
+	cUpper                        // 011
+	cIgnorableUncased             // 100
+	cIgnorableCased               // 101 // lower case if mappings exist
+	cXORCase                      // 11x // case is cLower | ((rune&1) ^ x)
+
+	maxCaseMode = cUpper
+)
+
+func (c info) isCased() bool {
+	return c&casedMask != 0
+}
+
+func (c info) isCaseIgnorable() bool {
+	return c&ignorableMask == ignorableValue
+}
+
+func (c info) isNotCasedAndNotCaseIgnorable() bool {
+	return c&fullCasedMask == 0
+}
+
+func (c info) isCaseIgnorableAndNotCased() bool {
+	return c&fullCasedMask == cIgnorableUncased
+}
+
+func (c info) isMid() bool {
+	return c&(fullCasedMask|isMidBit) == isMidBit|cIgnorableUncased
+}
+
+// The case mapping implementation will need to know about various Canonical
+// Combining Class (CCC) values. We encode two of these in the trie value:
+// cccZero (0) and cccAbove (230). If the value is cccOther, it means that
+// CCC(r) > 0, but not 230. A value of cccBreak means that CCC(r) == 0 and that
+// the rune also has the break category Break (see below).
+const (
+	cccBreak info = iota << 4
+	cccZero
+	cccAbove
+	cccOther
+
+	cccMask = cccBreak | cccZero | cccAbove | cccOther
+)
+
+const (
+	starter       = 0
+	above         = 230
+	iotaSubscript = 240
+)
+
+// The exceptions slice holds data that does not fit in a normal info entry.
+// The entry is pointed to by the exception index in an entry. It has the
+// following format:
+//
+// Header
+// byte 0:
+//  7..6  unused
+//  5..4  CCC type (same bits as entry)
+//     3  unused
+//  2..0  length of fold
+//
+// byte 1:
+//   7..6  unused
+//   5..3  length of 1st mapping of case type
+//   2..0  length of 2nd mapping of case type
+//
+//   case     1st    2nd
+//   lower -> upper, title
+//   upper -> lower, title
+//   title -> lower, upper
+//
+// Lengths with the value 0x7 indicate no value and implies no change.
+// A length of 0 indicates a mapping to zero-length string.
+//
+// Body bytes:
+//   case folding bytes
+//   lowercase mapping bytes
+//   uppercase mapping bytes
+//   titlecase mapping bytes
+//   closure mapping bytes (for NFKC_Casefold). (TODO)
+//
+// Fallbacks:
+//   missing fold  -> lower
+//   missing title -> upper
+//   all missing   -> original rune
+//
+// exceptions starts with a dummy byte to enforce that there is no zero index
+// value.
+const (
+	lengthMask = 0x07
+	lengthBits = 3
+	noChange   = 0
+)
+
+// References to generated trie.
+
+var trie = newCaseTrie(0)
+
+var sparse = sparseBlocks{
+	values:  sparseValues[:],
+	offsets: sparseOffsets[:],
+}
+
+// Sparse block lookup code.
+
+// valueRange is an entry in a sparse block.
+type valueRange struct {
+	value  uint16
+	lo, hi byte
+}
+
+type sparseBlocks struct {
+	values  []valueRange
+	offsets []uint16
+}
+
+// lookup returns the value from values block n for byte b using binary search.
+func (s *sparseBlocks) lookup(n uint32, b byte) uint16 {
+	lo := s.offsets[n]
+	hi := s.offsets[n+1]
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := s.values[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}
+
+// lastRuneForTesting is the last rune used for testing. Everything after this
+// is boring.
+const lastRuneForTesting = rune(0x1FFFF)

data/ext/vendor/golang.org/x/text/cases/icu.go

@@ -0,0 +1,61 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build icu
+
+package cases
+
+// Ideally these functions would be defined in a test file, but go test doesn't
+// allow CGO in tests. The build tag should ensure either way that these
+// functions will not end up in the package.
+
+// TODO: Ensure that the correct ICU version is set.
+
+/*
+#cgo LDFLAGS: -licui18n.57 -licuuc.57
+#include <stdlib.h>
+#include <unicode/ustring.h>
+#include <unicode/utypes.h>
+#include <unicode/localpointer.h>
+#include <unicode/ucasemap.h>
+*/
+import "C"
+
+import "unsafe"
+
+func doICU(tag, caser, input string) string {
+	err := C.UErrorCode(0)
+	loc := C.CString(tag)
+	cm := C.ucasemap_open(loc, C.uint32_t(0), &err)
+
+	buf := make([]byte, len(input)*4)
+	dst := (*C.char)(unsafe.Pointer(&buf[0]))
+	src := C.CString(input)
+
+	cn := C.int32_t(0)
+
+	switch caser {
+	case "fold":
+		cn = C.ucasemap_utf8FoldCase(cm,
+			dst, C.int32_t(len(buf)),
+			src, C.int32_t(len(input)),
+			&err)
+	case "lower":
+		cn = C.ucasemap_utf8ToLower(cm,
+			dst, C.int32_t(len(buf)),
+			src, C.int32_t(len(input)),
+			&err)
+	case "upper":
+		cn = C.ucasemap_utf8ToUpper(cm,
+			dst, C.int32_t(len(buf)),
+			src, C.int32_t(len(input)),
+			&err)
+	case "title":
+		cn = C.ucasemap_utf8ToTitle(cm,
+			dst, C.int32_t(len(buf)),
+			src, C.int32_t(len(input)),
+			&err)
+	}
+	return string(buf[:cn])
+}

data/ext/vendor/golang.org/x/text/cases/icu_test.go

@@ -0,0 +1,210 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build icu
+
+package cases
+
+import (
+	"path"
+	"strings"
+	"testing"
+
+	"golang.org/x/text/internal/testtext"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/norm"
+)
+
+func TestICUConformance(t *testing.T) {
+	// Build test set.
+	input := []string{
+		"a.a a_a",
+		"a\u05d0a",
+		"\u05d0'a",
+		"a\u03084a",
+		"a\u0308a",
+		"a3\u30a3a",
+		"a\u303aa",
+		"a_\u303a_a",
+		"1_a..a",
+		"1_a.a",
+		"a..a.",
+		"a--a-",
+		"a-a-",
+		"a\u200ba",
+		"a\u200b\u200ba",
+		"a\u00ad\u00ada", // Format
+		"a\u00ada",
+		"a''a", // SingleQuote
+		"a'a",
+		"a::a", // MidLetter
+		"a:a",
+		"a..a", // MidNumLet
+		"a.a",
+		"a;;a", // MidNum
+		"a;a",
+		"a__a", // ExtendNumlet
+		"a_a",
+		"ΟΣ''a",
+	}
+	add := func(x interface{}) {
+		switch v := x.(type) {
+		case string:
+			input = append(input, v)
+		case []string:
+			for _, s := range v {
+				input = append(input, s)
+			}
+		}
+	}
+	for _, tc := range testCases {
+		add(tc.src)
+		add(tc.lower)
+		add(tc.upper)
+		add(tc.title)
+	}
+	for _, tc := range bufferTests {
+		add(tc.src)
+	}
+	for _, tc := range breakTest {
+		add(strings.Replace(tc, "|", "", -1))
+	}
+	for _, tc := range foldTestCases {
+		add(tc)
+	}
+
+	// Compare ICU to Go.
+	for _, c := range []string{"lower", "upper", "title", "fold"} {
+		for _, tag := range []string{
+			"und", "af", "az", "el", "lt", "nl", "tr",
+		} {
+			for _, s := range input {
+				if exclude(c, tag, s) {
+					continue
+				}
+				testtext.Run(t, path.Join(c, tag, s), func(t *testing.T) {
+					want := doICU(tag, c, s)
+					got := doGo(tag, c, s)
+					if norm.NFC.String(got) != norm.NFC.String(want) {
+						t.Errorf("\n    in %[3]q (%+[3]q)\n   got %[1]q (%+[1]q)\n  want %[2]q (%+[2]q)", got, want, s)
+					}
+				})
+			}
+		}
+	}
+}
+
+// exclude indicates if a string should be excluded from testing.
+func exclude(cm, tag, s string) bool {
+	list := []struct{ cm, tags, pattern string }{
+		// TODO: Go does not handle certain esoteric breaks correctly. This will be
+		// fixed once we have a real word break iterator. Alternatively, it
+		// seems like we're not too far off from making it work, so we could
+		// fix these last steps. But first verify that using a separate word
+		// breaker does not hurt performance.
+		{"title", "af nl", "a''a"},
+		{"", "", "א'a"},
+
+		// All the exclusions below seem to be issues with the ICU
+		// implementation (at version 57) and thus are not marked as TODO.
+
+		// ICU does not handle leading apostrophe for Dutch and
+		// Afrikaans correctly. See http://unicode.org/cldr/trac/ticket/7078.
+		{"title", "af nl", "'n"},
+		{"title", "af nl", "'N"},
+
+		// Go terminates the final sigma check after a fixed number of
+		// ignorables have been found. This ensures that the algorithm can make
+		// progress in a streaming scenario.
+		{"lower title", "", "\u039f\u03a3...............................a"},
+		// This also applies to upper in Greek.
+		// NOTE: we could fix the following two cases by adding state to elUpper
+		// and aztrLower. However, considering a modifier to not belong to the
+		// preceding letter after the maximum modifiers count is reached is
+		// consistent with the behavior of unicode/norm.
+		{"upper", "el", "\u03bf" + strings.Repeat("\u0321", 29) + "\u0313"},
+		{"lower", "az tr lt", "I" + strings.Repeat("\u0321", 30) + "\u0307\u0300"},
+		{"upper", "lt", "i" + strings.Repeat("\u0321", 30) + "\u0307\u0300"},
+		{"lower", "lt", "I" + strings.Repeat("\u0321", 30) + "\u0300"},
+
+		// ICU title case seems to erroneously removes \u0307 from an upper case
+		// I unconditionally, instead of only when lowercasing. The ICU
+		// transform algorithm transforms these cases consistently with our
+		// implementation.
+		{"title", "az tr", "\u0307"},
+
+		// The spec says to remove \u0307 after Soft-Dotted characters. ICU
+		// transforms conform but ucasemap_utf8ToUpper does not.
+		{"upper title", "lt", "i\u0307"},
+		{"upper title", "lt", "i" + strings.Repeat("\u0321", 29) + "\u0307\u0300"},
+
+		// Both Unicode and CLDR prescribe an extra explicit dot above after a
+		// Soft_Dotted character if there are other modifiers.
+		// ucasemap_utf8ToUpper does not do this; ICU transforms do.
+		// The issue with ucasemap_utf8ToUpper seems to be that it does not
+		// consider the modifiers that are part of composition in the evaluation
+		// of More_Above. For instance, according to the More_Above rule for lt,
+		// a dotted capital I (U+0130) becomes i\u0307\u0307 (an small i with
+		// two additional dots). This seems odd, but is correct. ICU is
+		// definitely not correct as it produces different results for different
+		// normal forms. For instance, for an İ:
+		//    \u0130  (NFC) -> i\u0307         (incorrect)
+		//    I\u0307 (NFD) -> i\u0307\u0307   (correct)
+		// We could argue that we should not add a \u0307 if there already is
+		// one, but this may be hard to get correct and is not conform the
+		// standard.
+		{"lower title", "lt", "\u0130"},
+		{"lower title", "lt", "\u00cf"},
+
+		// We are conform ICU ucasemap_utf8ToUpper if we remove support for
+		// elUpper. However, this is clearly not conform the spec. Moreover, the
+		// ICU transforms _do_ implement this transform and produces results
+		// consistent with our implementation. Note that we still prefer to use
+		// ucasemap_utf8ToUpper instead of transforms as the latter have
+		// inconsistencies in the word breaking algorithm.
+		{"upper", "el", "\u0386"}, // GREEK CAPITAL LETTER ALPHA WITH TONOS
+		{"upper", "el", "\u0389"}, // GREEK CAPITAL LETTER ETA WITH TONOS
+		{"upper", "el", "\u038A"}, // GREEK CAPITAL LETTER IOTA WITH TONOS
+
+		{"upper", "el", "\u0391"}, // GREEK CAPITAL LETTER ALPHA
+		{"upper", "el", "\u0397"}, // GREEK CAPITAL LETTER ETA
+		{"upper", "el", "\u0399"}, // GREEK CAPITAL LETTER IOTA
+
+		{"upper", "el", "\u03AC"}, // GREEK SMALL LETTER ALPHA WITH TONOS
+		{"upper", "el", "\u03AE"}, // GREEK SMALL LETTER ALPHA WITH ETA
+		{"upper", "el", "\u03AF"}, // GREEK SMALL LETTER ALPHA WITH IOTA
+
+		{"upper", "el", "\u03B1"}, // GREEK SMALL LETTER ALPHA
+		{"upper", "el", "\u03B7"}, // GREEK SMALL LETTER ETA
+		{"upper", "el", "\u03B9"}, // GREEK SMALL LETTER IOTA
+	}
+	for _, x := range list {
+		if x.cm != "" && strings.Index(x.cm, cm) == -1 {
+			continue
+		}
+		if x.tags != "" && strings.Index(x.tags, tag) == -1 {
+			continue
+		}
+		if strings.Index(s, x.pattern) != -1 {
+			return true
+		}
+	}
+	return false
+}
+
+func doGo(tag, caser, input string) string {
+	var c Caser
+	t := language.MustParse(tag)
+	switch caser {
+	case "lower":
+		c = Lower(t)
+	case "upper":
+		c = Upper(t)
+	case "title":
+		c = Title(t)
+	case "fold":
+		c = Fold()
+	}
+	return c.String(input)
+}