ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/collate/option.go

@@ -0,0 +1,239 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"sort"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/norm"
+)
+
+// newCollator creates a new collator with default options configured.
+func newCollator(t colltab.Weighter) *Collator {
+	// Initialize a collator with default options.
+	c := &Collator{
+		options: options{
+			ignore: [colltab.NumLevels]bool{
+				colltab.Quaternary: true,
+				colltab.Identity:   true,
+			},
+			f: norm.NFD,
+			t: t,
+		},
+	}
+
+	// TODO: store vt in tags or remove.
+	c.variableTop = t.Top()
+
+	return c
+}
+
+// An Option is used to change the behavior of a Collator. Options override the
+// settings passed through the locale identifier.
+type Option struct {
+	priority int
+	f        func(o *options)
+}
+
+type prioritizedOptions []Option
+
+func (p prioritizedOptions) Len() int {
+	return len(p)
+}
+
+func (p prioritizedOptions) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+func (p prioritizedOptions) Less(i, j int) bool {
+	return p[i].priority < p[j].priority
+}
+
+type options struct {
+	// ignore specifies which levels to ignore.
+	ignore [colltab.NumLevels]bool
+
+	// caseLevel is true if there is an additional level of case matching
+	// between the secondary and tertiary levels.
+	caseLevel bool
+
+	// backwards specifies the order of sorting at the secondary level.
+	// This option exists predominantly to support reverse sorting of accents in French.
+	backwards bool
+
+	// numeric specifies whether any sequence of decimal digits (category is Nd)
+	// is sorted at a primary level with its numeric value.
+	// For example, "A-21" < "A-123".
+	// This option is set by wrapping the main Weighter with NewNumericWeighter.
+	numeric bool
+
+	// alternate specifies an alternative handling of variables.
+	alternate alternateHandling
+
+	// variableTop is the largest primary value that is considered to be
+	// variable.
+	variableTop uint32
+
+	t colltab.Weighter
+
+	f norm.Form
+}
+
+func (o *options) setOptions(opts []Option) {
+	sort.Sort(prioritizedOptions(opts))
+	for _, x := range opts {
+		x.f(o)
+	}
+}
+
+// OptionsFromTag extracts the BCP47 collation options from the tag and
+// configures a collator accordingly. These options are set before any other
+// option.
+func OptionsFromTag(t language.Tag) Option {
+	return Option{0, func(o *options) {
+		o.setFromTag(t)
+	}}
+}
+
+func (o *options) setFromTag(t language.Tag) {
+	o.caseLevel = ldmlBool(t, o.caseLevel, "kc")
+	o.backwards = ldmlBool(t, o.backwards, "kb")
+	o.numeric = ldmlBool(t, o.numeric, "kn")
+
+	// Extract settings from the BCP47 u extension.
+	switch t.TypeForKey("ks") { // strength
+	case "level1":
+		o.ignore[colltab.Secondary] = true
+		o.ignore[colltab.Tertiary] = true
+	case "level2":
+		o.ignore[colltab.Tertiary] = true
+	case "level3", "":
+		// The default.
+	case "level4":
+		o.ignore[colltab.Quaternary] = false
+	case "identic":
+		o.ignore[colltab.Quaternary] = false
+		o.ignore[colltab.Identity] = false
+	}
+
+	switch t.TypeForKey("ka") {
+	case "shifted":
+		o.alternate = altShifted
+	// The following two types are not official BCP47, but we support them to
+	// give access to this otherwise hidden functionality. The name blanked is
+	// derived from the LDML name blanked and posix reflects the main use of
+	// the shift-trimmed option.
+	case "blanked":
+		o.alternate = altBlanked
+	case "posix":
+		o.alternate = altShiftTrimmed
+	}
+
+	// TODO: caseFirst ("kf"), reorder ("kr"), and maybe variableTop ("vt").
+
+	// Not used:
+	// - normalization ("kk", not necessary for this implementation)
+	// - hiraganaQuatenary ("kh", obsolete)
+}
+
+func ldmlBool(t language.Tag, old bool, key string) bool {
+	switch t.TypeForKey(key) {
+	case "true":
+		return true
+	case "false":
+		return false
+	default:
+		return old
+	}
+}
+
+var (
+	// IgnoreCase sets case-insensitive comparison.
+	IgnoreCase Option = ignoreCase
+	ignoreCase        = Option{3, ignoreCaseF}
+
+	// IgnoreDiacritics causes diacritical marks to be ignored. ("o" == "ö").
+	IgnoreDiacritics Option = ignoreDiacritics
+	ignoreDiacritics        = Option{3, ignoreDiacriticsF}
+
+	// IgnoreWidth causes full-width characters to match their half-width
+	// equivalents.
+	IgnoreWidth Option = ignoreWidth
+	ignoreWidth        = Option{2, ignoreWidthF}
+
+	// Loose sets the collator to ignore diacritics, case and weight.
+	Loose Option = loose
+	loose        = Option{4, looseF}
+
+	// Force ordering if strings are equivalent but not equal.
+	Force Option = force
+	force        = Option{5, forceF}
+
+	// Numeric specifies that numbers should sort numerically ("2" < "12").
+	Numeric Option = numeric
+	numeric        = Option{5, numericF}
+)
+
+func ignoreWidthF(o *options) {
+	o.ignore[colltab.Tertiary] = true
+	o.caseLevel = true
+}
+
+func ignoreDiacriticsF(o *options) {
+	o.ignore[colltab.Secondary] = true
+}
+
+func ignoreCaseF(o *options) {
+	o.ignore[colltab.Tertiary] = true
+	o.caseLevel = false
+}
+
+func looseF(o *options) {
+	ignoreWidthF(o)
+	ignoreDiacriticsF(o)
+	ignoreCaseF(o)
+}
+
+func forceF(o *options) {
+	o.ignore[colltab.Identity] = false
+}
+
+func numericF(o *options) { o.numeric = true }
+
+// Reorder overrides the pre-defined ordering of scripts and character sets.
+func Reorder(s ...string) Option {
+	// TODO: need fractional weights to implement this.
+	panic("TODO: implement")
+}
+
+// TODO: consider making these public again. These options cannot be fully
+// specified in BCP47, so an API interface seems warranted. Still a higher-level
+// interface would be nice (e.g. a POSIX option for enabling altShiftTrimmed)
+
+// alternateHandling identifies the various ways in which variables are handled.
+// A rune with a primary weight lower than the variable top is considered a
+// variable.
+// See http://www.unicode.org/reports/tr10/#Variable_Weighting for details.
+type alternateHandling int
+
+const (
+	// altNonIgnorable turns off special handling of variables.
+	altNonIgnorable alternateHandling = iota
+
+	// altBlanked sets variables and all subsequent primary ignorables to be
+	// ignorable at all levels. This is identical to removing all variables
+	// and subsequent primary ignorables from the input.
+	altBlanked
+
+	// altShifted sets variables to be ignorable for levels one through three and
+	// adds a fourth level based on the values of the ignored levels.
+	altShifted
+
+	// altShiftTrimmed is a slight variant of altShifted that is used to
+	// emulate POSIX.
+	altShiftTrimmed
+)

data/ext/vendor/golang.org/x/text/collate/option_test.go

@@ -0,0 +1,209 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+package collate
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+)
+
+var (
+	defaultIgnore = ignore(colltab.Tertiary)
+	defaultTable  = getTable(locales[0])
+)
+
+func TestOptions(t *testing.T) {
+	for i, tt := range []struct {
+		in  []Option
+		out options
+	}{
+		0: {
+			out: options{
+				ignore: defaultIgnore,
+			},
+		},
+		1: {
+			in: []Option{IgnoreDiacritics},
+			out: options{
+				ignore: [colltab.NumLevels]bool{false, true, false, true, true},
+			},
+		},
+		2: {
+			in: []Option{IgnoreCase, IgnoreDiacritics},
+			out: options{
+				ignore: ignore(colltab.Primary),
+			},
+		},
+		3: {
+			in: []Option{ignoreDiacritics, IgnoreWidth},
+			out: options{
+				ignore:    ignore(colltab.Primary),
+				caseLevel: true,
+			},
+		},
+		4: {
+			in: []Option{IgnoreWidth, ignoreDiacritics},
+			out: options{
+				ignore:    ignore(colltab.Primary),
+				caseLevel: true,
+			},
+		},
+		5: {
+			in: []Option{IgnoreCase, IgnoreWidth},
+			out: options{
+				ignore: ignore(colltab.Secondary),
+			},
+		},
+		6: {
+			in: []Option{IgnoreCase, IgnoreWidth, Loose},
+			out: options{
+				ignore: ignore(colltab.Primary),
+			},
+		},
+		7: {
+			in: []Option{Force, IgnoreCase, IgnoreWidth, Loose},
+			out: options{
+				ignore: [colltab.NumLevels]bool{false, true, true, true, false},
+			},
+		},
+		8: {
+			in: []Option{IgnoreDiacritics, IgnoreCase},
+			out: options{
+				ignore: ignore(colltab.Primary),
+			},
+		},
+		9: {
+			in: []Option{Numeric},
+			out: options{
+				ignore:  defaultIgnore,
+				numeric: true,
+			},
+		},
+		10: {
+			in: []Option{OptionsFromTag(language.MustParse("und-u-ks-level1"))},
+			out: options{
+				ignore: ignore(colltab.Primary),
+			},
+		},
+		11: {
+			in: []Option{OptionsFromTag(language.MustParse("und-u-ks-level4"))},
+			out: options{
+				ignore: ignore(colltab.Quaternary),
+			},
+		},
+		12: {
+			in:  []Option{OptionsFromTag(language.MustParse("und-u-ks-identic"))},
+			out: options{},
+		},
+		13: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-kn-true-kb-true-kc-true")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				caseLevel: true,
+				backwards: true,
+				numeric:   true,
+			},
+		},
+		14: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-kn-true-kb-true-kc-true")),
+				OptionsFromTag(language.MustParse("und-u-kn-false-kb-false-kc-false")),
+			},
+			out: options{
+				ignore: defaultIgnore,
+			},
+		},
+		15: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-kn-true-kb-true-kc-true")),
+				OptionsFromTag(language.MustParse("und-u-kn-foo-kb-foo-kc-foo")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				caseLevel: true,
+				backwards: true,
+				numeric:   true,
+			},
+		},
+		16: { // Normal options take precedence over tag options.
+			in: []Option{
+				Numeric, IgnoreCase,
+				OptionsFromTag(language.MustParse("und-u-kn-false-kc-true")),
+			},
+			out: options{
+				ignore:    ignore(colltab.Secondary),
+				caseLevel: false,
+				numeric:   true,
+			},
+		},
+		17: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-ka-shifted")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				alternate: altShifted,
+			},
+		},
+		18: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-ka-blanked")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				alternate: altBlanked,
+			},
+		},
+		19: {
+			in: []Option{
+				OptionsFromTag(language.MustParse("und-u-ka-posix")),
+			},
+			out: options{
+				ignore:    defaultIgnore,
+				alternate: altShiftTrimmed,
+			},
+		},
+	} {
+		c := newCollator(defaultTable)
+		c.t = nil
+		c.variableTop = 0
+		c.f = 0
+
+		c.setOptions(tt.in)
+		if !reflect.DeepEqual(c.options, tt.out) {
+			t.Errorf("%d: got %v; want %v", i, c.options, tt.out)
+		}
+	}
+}
+
+func TestAlternateSortTypes(t *testing.T) {
+	testCases := []struct {
+		lang string
+		in   []string
+		want []string
+	}{{
+		lang: "zh,cmn,zh-Hant-u-co-pinyin,zh-HK-u-co-pinyin,zh-pinyin",
+		in:   []string{"爸爸", "妈妈", "儿子", "女儿"},
+		want: []string{"爸爸", "儿子", "妈妈", "女儿"},
+	}, {
+		lang: "zh-Hant,zh-u-co-stroke,zh-Hant-u-co-stroke",
+		in:   []string{"爸爸", "妈妈", "儿子", "女儿"},
+		want: []string{"儿子", "女儿", "妈妈", "爸爸"},
+	}}
+	for _, tc := range testCases {
+		for _, tag := range strings.Split(tc.lang, ",") {
+			got := append([]string{}, tc.in...)
+			New(language.MustParse(tag)).SortStrings(got)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("New(%s).SortStrings(%v) = %v; want %v", tag, tc.in, got, tc.want)
+			}
+		}
+	}
+}

data/ext/vendor/golang.org/x/text/collate/reg_test.go

@@ -0,0 +1,230 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"flag"
+	"io"
+	"io/ioutil"
+	"log"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	"golang.org/x/text/collate/build"
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/language"
+)
+
+var long = flag.Bool("long", false,
+	"run time-consuming tests, such as tests that fetch data online")
+
+// This regression test runs tests for the test files in CollationTest.zip
+// (taken from http://www.unicode.org/Public/UCA/<gen.UnicodeVersion()>/).
+//
+// The test files have the following form:
+// # header
+// 0009 0021;	# ('\u0009') <CHARACTER TABULATION>	[| | | 0201 025E]
+// 0009 003F;	# ('\u0009') <CHARACTER TABULATION>	[| | | 0201 0263]
+// 000A 0021;	# ('\u000A') <LINE FEED (LF)>	[| | | 0202 025E]
+// 000A 003F;	# ('\u000A') <LINE FEED (LF)>	[| | | 0202 0263]
+//
+// The part before the semicolon is the hex representation of a sequence
+// of runes. After the hash mark is a comment. The strings
+// represented by rune sequence are in the file in sorted order, as
+// defined by the DUCET.
+
+type Test struct {
+	name    string
+	str     [][]byte
+	comment []string
+}
+
+var versionRe = regexp.MustCompile(`# UCA Version: (.*)\n?$`)
+var testRe = regexp.MustCompile(`^([\dA-F ]+);.*# (.*)\n?$`)
+
+func TestCollation(t *testing.T) {
+	if !gen.IsLocal() && !*long {
+		t.Skip("skipping test to prevent downloading; to run use -long or use -local to specify a local source")
+	}
+	t.Skip("must first update to new file format to support test")
+	for _, test := range loadTestData() {
+		doTest(t, test)
+	}
+}
+
+func Error(e error) {
+	if e != nil {
+		log.Fatal(e)
+	}
+}
+
+// parseUCA parses a Default Unicode Collation Element Table of the format
+// specified in http://www.unicode.org/reports/tr10/#File_Format.
+// It returns the variable top.
+func parseUCA(builder *build.Builder) {
+	r := gen.OpenUnicodeFile("UCA", "", "allkeys.txt")
+	defer r.Close()
+	input := bufio.NewReader(r)
+	colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
+	for i := 1; true; i++ {
+		l, prefix, err := input.ReadLine()
+		if err == io.EOF {
+			break
+		}
+		Error(err)
+		line := string(l)
+		if prefix {
+			log.Fatalf("%d: buffer overflow", i)
+		}
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+		if line[0] == '@' {
+			if strings.HasPrefix(line[1:], "version ") {
+				if v := strings.Split(line[1:], " ")[1]; v != gen.UnicodeVersion() {
+					log.Fatalf("incompatible version %s; want %s", v, gen.UnicodeVersion())
+				}
+			}
+		} else {
+			// parse entries
+			part := strings.Split(line, " ; ")
+			if len(part) != 2 {
+				log.Fatalf("%d: production rule without ';': %v", i, line)
+			}
+			lhs := []rune{}
+			for _, v := range strings.Split(part[0], " ") {
+				if v != "" {
+					lhs = append(lhs, rune(convHex(i, v)))
+				}
+			}
+			vars := []int{}
+			rhs := [][]int{}
+			for i, m := range colelem.FindAllStringSubmatch(part[1], -1) {
+				if m[1] == "*" {
+					vars = append(vars, i)
+				}
+				elem := []int{}
+				for _, h := range strings.Split(m[2], ".") {
+					elem = append(elem, convHex(i, h))
+				}
+				rhs = append(rhs, elem)
+			}
+			builder.Add(lhs, rhs, vars)
+		}
+	}
+}
+
+func convHex(line int, s string) int {
+	r, e := strconv.ParseInt(s, 16, 32)
+	if e != nil {
+		log.Fatalf("%d: %v", line, e)
+	}
+	return int(r)
+}
+
+func loadTestData() []Test {
+	f := gen.OpenUnicodeFile("UCA", "", "CollationTest.zip")
+	buffer, err := ioutil.ReadAll(f)
+	f.Close()
+	Error(err)
+	archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+	Error(err)
+	tests := []Test{}
+	for _, f := range archive.File {
+		// Skip the short versions, which are simply duplicates of the long versions.
+		if strings.Contains(f.Name, "SHORT") || f.FileInfo().IsDir() {
+			continue
+		}
+		ff, err := f.Open()
+		Error(err)
+		defer ff.Close()
+		scanner := bufio.NewScanner(ff)
+		test := Test{name: path.Base(f.Name)}
+		for scanner.Scan() {
+			line := scanner.Text()
+			if len(line) <= 1 || line[0] == '#' {
+				if m := versionRe.FindStringSubmatch(line); m != nil {
+					if m[1] != gen.UnicodeVersion() {
+						log.Printf("warning:%s: version is %s; want %s", f.Name, m[1], gen.UnicodeVersion())
+					}
+				}
+				continue
+			}
+			m := testRe.FindStringSubmatch(line)
+			if m == nil || len(m) < 3 {
+				log.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
+			}
+			str := []byte{}
+			// In the regression test data (unpaired) surrogates are assigned a weight
+			// corresponding to their code point value.  However, utf8.DecodeRune,
+			// which is used to compute the implicit weight, assigns FFFD to surrogates.
+			// We therefore skip tests with surrogates.  This skips about 35 entries
+			// per test.
+			valid := true
+			for _, split := range strings.Split(m[1], " ") {
+				r, err := strconv.ParseUint(split, 16, 64)
+				Error(err)
+				valid = valid && utf8.ValidRune(rune(r))
+				str = append(str, string(rune(r))...)
+			}
+			if valid {
+				test.str = append(test.str, str)
+				test.comment = append(test.comment, m[2])
+			}
+		}
+		if scanner.Err() != nil {
+			log.Fatal(scanner.Err())
+		}
+		tests = append(tests, test)
+	}
+	return tests
+}
+
+var errorCount int
+
+func runes(b []byte) []rune {
+	return []rune(string(b))
+}
+
+var shifted = language.MustParse("und-u-ka-shifted-ks-level4")
+
+func doTest(t *testing.T, tc Test) {
+	bld := build.NewBuilder()
+	parseUCA(bld)
+	w, err := bld.Build()
+	Error(err)
+	var tag language.Tag
+	if !strings.Contains(tc.name, "NON_IGNOR") {
+		tag = shifted
+	}
+	c := NewFromTable(w, OptionsFromTag(tag))
+	b := &Buffer{}
+	prev := tc.str[0]
+	for i := 1; i < len(tc.str); i++ {
+		b.Reset()
+		s := tc.str[i]
+		ka := c.Key(b, prev)
+		kb := c.Key(b, s)
+		if r := bytes.Compare(ka, kb); r == 1 {
+			t.Errorf("%s:%d: Key(%.4X) < Key(%.4X) (%X < %X) == %d; want -1 or 0", tc.name, i, []rune(string(prev)), []rune(string(s)), ka, kb, r)
+			prev = s
+			continue
+		}
+		if r := c.Compare(prev, s); r == 1 {
+			t.Errorf("%s:%d: Compare(%.4X, %.4X) == %d; want -1 or 0", tc.name, i, runes(prev), runes(s), r)
+		}
+		if r := c.Compare(s, prev); r == -1 {
+			t.Errorf("%s:%d: Compare(%.4X, %.4X) == %d; want 1 or 0", tc.name, i, runes(s), runes(prev), r)
+		}
+		prev = s
+	}
+}