ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/internal/colltab/contract_test.go

@@ -0,0 +1,131 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"testing"
+)
+
+type lookupStrings struct {
+	str    string
+	offset int
+	n      int // bytes consumed from input
+}
+
+type LookupTest struct {
+	lookup []lookupStrings
+	n      int
+	tries  ContractTrieSet
+}
+
+var lookupTests = []LookupTest{{
+	[]lookupStrings{
+		{"abc", 1, 3},
+		{"a", 0, 0},
+		{"b", 0, 0},
+		{"c", 0, 0},
+		{"d", 0, 0},
+	},
+	1,
+	ContractTrieSet{
+		{'a', 0, 1, 0xFF},
+		{'b', 0, 1, 0xFF},
+		{'c', 'c', 0, 1},
+	},
+}, {
+	[]lookupStrings{
+		{"abc", 1, 3},
+		{"abd", 2, 3},
+		{"abe", 3, 3},
+		{"a", 0, 0},
+		{"ab", 0, 0},
+		{"d", 0, 0},
+		{"f", 0, 0},
+	},
+	1,
+	ContractTrieSet{
+		{'a', 0, 1, 0xFF},
+		{'b', 0, 1, 0xFF},
+		{'c', 'e', 0, 1},
+	},
+}, {
+	[]lookupStrings{
+		{"abc", 1, 3},
+		{"ab", 2, 2},
+		{"a", 3, 1},
+		{"abcd", 1, 3},
+		{"abe", 2, 2},
+	},
+	1,
+	ContractTrieSet{
+		{'a', 0, 1, 3},
+		{'b', 0, 1, 2},
+		{'c', 'c', 0, 1},
+	},
+}, {
+	[]lookupStrings{
+		{"abc", 1, 3},
+		{"abd", 2, 3},
+		{"ab", 3, 2},
+		{"ac", 4, 2},
+		{"a", 5, 1},
+		{"b", 6, 1},
+		{"ba", 6, 1},
+	},
+	2,
+	ContractTrieSet{
+		{'b', 'b', 0, 6},
+		{'a', 0, 2, 5},
+		{'c', 'c', 0, 4},
+		{'b', 0, 1, 3},
+		{'c', 'd', 0, 1},
+	},
+}, {
+	[]lookupStrings{
+		{"bcde", 2, 4},
+		{"bc", 7, 2},
+		{"ab", 6, 2},
+		{"bcd", 5, 3},
+		{"abcd", 1, 4},
+		{"abc", 4, 3},
+		{"bcdf", 3, 4},
+	},
+	2,
+	ContractTrieSet{
+		{'b', 3, 1, 0xFF},
+		{'a', 0, 1, 0xFF},
+		{'b', 0, 1, 6},
+		{'c', 0, 1, 4},
+		{'d', 'd', 0, 1},
+		{'c', 0, 1, 7},
+		{'d', 0, 1, 5},
+		{'e', 'f', 0, 2},
+	},
+}}
+
+func lookup(c *ContractTrieSet, nnode int, s []uint8) (i, n int) {
+	scan := c.scanner(0, nnode, s)
+	scan.scan(0)
+	return scan.result()
+}
+
+func TestLookupContraction(t *testing.T) {
+	for i, tt := range lookupTests {
+		cts := ContractTrieSet(tt.tries)
+		for j, lu := range tt.lookup {
+			str := lu.str
+			for _, s := range []string{str, str + "X"} {
+				const msg = `%d:%d: %s of "%s" %v; want %v`
+				offset, n := lookup(&cts, tt.n, []byte(s))
+				if offset != lu.offset {
+					t.Errorf(msg, i, j, "offset", s, offset, lu.offset)
+				}
+				if n != lu.n {
+					t.Errorf(msg, i, j, "bytes consumed", s, n, len(str))
+				}
+			}
+		}
+	}
+}

data/ext/vendor/golang.org/x/text/internal/colltab/iter.go

@@ -0,0 +1,178 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+// An Iter incrementally converts chunks of the input text to collation
+// elements, while ensuring that the collation elements are in normalized order
+// (that is, they are in the order as if the input text were normalized first).
+type Iter struct {
+	Weighter Weighter
+	Elems    []Elem
+	// N is the number of elements in Elems that will not be reordered on
+	// subsequent iterations, N <= len(Elems).
+	N int
+
+	bytes []byte
+	str   string
+	// Because the Elems buffer may contain collation elements that are needed
+	// for look-ahead, we need two positions in the text (bytes or str): one for
+	// the end position in the text for the current iteration and one for the
+	// start of the next call to appendNext.
+	pEnd  int // end position in text corresponding to N.
+	pNext int // pEnd <= pNext.
+}
+
+// Reset sets the position in the current input text to p and discards any
+// results obtained so far.
+func (i *Iter) Reset(p int) {
+	i.Elems = i.Elems[:0]
+	i.N = 0
+	i.pEnd = p
+	i.pNext = p
+}
+
+// Len returns the length of the input text.
+func (i *Iter) Len() int {
+	if i.bytes != nil {
+		return len(i.bytes)
+	}
+	return len(i.str)
+}
+
+// Discard removes the collation elements up to N.
+func (i *Iter) Discard() {
+	// TODO: change this such that only modifiers following starters will have
+	// to be copied.
+	i.Elems = i.Elems[:copy(i.Elems, i.Elems[i.N:])]
+	i.N = 0
+}
+
+// End returns the end position of the input text for which Next has returned
+// results.
+func (i *Iter) End() int {
+	return i.pEnd
+}
+
+// SetInput resets i to input s.
+func (i *Iter) SetInput(s []byte) {
+	i.bytes = s
+	i.str = ""
+	i.Reset(0)
+}
+
+// SetInputString resets i to input s.
+func (i *Iter) SetInputString(s string) {
+	i.str = s
+	i.bytes = nil
+	i.Reset(0)
+}
+
+func (i *Iter) done() bool {
+	return i.pNext >= len(i.str) && i.pNext >= len(i.bytes)
+}
+
+func (i *Iter) appendNext() bool {
+	if i.done() {
+		return false
+	}
+	var sz int
+	if i.bytes == nil {
+		i.Elems, sz = i.Weighter.AppendNextString(i.Elems, i.str[i.pNext:])
+	} else {
+		i.Elems, sz = i.Weighter.AppendNext(i.Elems, i.bytes[i.pNext:])
+	}
+	if sz == 0 {
+		sz = 1
+	}
+	i.pNext += sz
+	return true
+}
+
+// Next appends Elems to the internal array. On each iteration, it will either
+// add starters or modifiers. In the majority of cases, an Elem with a primary
+// value > 0 will have a CCC of 0. The CCC values of collation elements are also
+// used to detect if the input string was not normalized and to adjust the
+// result accordingly.
+func (i *Iter) Next() bool {
+	if i.N == len(i.Elems) && !i.appendNext() {
+		return false
+	}
+
+	// Check if the current segment starts with a starter.
+	prevCCC := i.Elems[len(i.Elems)-1].CCC()
+	if prevCCC == 0 {
+		i.N = len(i.Elems)
+		i.pEnd = i.pNext
+		return true
+	} else if i.Elems[i.N].CCC() == 0 {
+		// set i.N to only cover part of i.Elems for which prevCCC == 0 and
+		// use rest for the next call to next.
+		for i.N++; i.N < len(i.Elems) && i.Elems[i.N].CCC() == 0; i.N++ {
+		}
+		i.pEnd = i.pNext
+		return true
+	}
+
+	// The current (partial) segment starts with modifiers. We need to collect
+	// all successive modifiers to ensure that they are normalized.
+	for {
+		p := len(i.Elems)
+		i.pEnd = i.pNext
+		if !i.appendNext() {
+			break
+		}
+
+		if ccc := i.Elems[p].CCC(); ccc == 0 || len(i.Elems)-i.N > maxCombiningCharacters {
+			// Leave the starter for the next iteration. This ensures that we
+			// do not return sequences of collation elements that cross two
+			// segments.
+			//
+			// TODO: handle large number of combining characters by fully
+			// normalizing the input segment before iteration. This ensures
+			// results are consistent across the text repo.
+			i.N = p
+			return true
+		} else if ccc < prevCCC {
+			i.doNorm(p, ccc) // should be rare, never occurs for NFD and FCC.
+		} else {
+			prevCCC = ccc
+		}
+	}
+
+	done := len(i.Elems) != i.N
+	i.N = len(i.Elems)
+	return done
+}
+
+// nextNoNorm is the same as next, but does not "normalize" the collation
+// elements.
+func (i *Iter) nextNoNorm() bool {
+	// TODO: remove this function. Using this instead of next does not seem
+	// to improve performance in any significant way. We retain this until
+	// later for evaluation purposes.
+	if i.done() {
+		return false
+	}
+	i.appendNext()
+	i.N = len(i.Elems)
+	return true
+}
+
+const maxCombiningCharacters = 30
+
+// doNorm reorders the collation elements in i.Elems.
+// It assumes that blocks of collation elements added with appendNext
+// either start and end with the same CCC or start with CCC == 0.
+// This allows for a single insertion point for the entire block.
+// The correctness of this assumption is verified in builder.go.
+func (i *Iter) doNorm(p int, ccc uint8) {
+	n := len(i.Elems)
+	k := p
+	for p--; p > i.N && ccc < i.Elems[p-1].CCC(); p-- {
+	}
+	i.Elems = append(i.Elems, i.Elems[p:k]...)
+	copy(i.Elems[p:], i.Elems[k:])
+	i.Elems = i.Elems[:n]
+}

data/ext/vendor/golang.org/x/text/internal/colltab/iter_test.go

@@ -0,0 +1,63 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"testing"
+)
+
+func TestDoNorm(t *testing.T) {
+	const div = -1 // The insertion point of the next block.
+	tests := []struct {
+		in, out []int
+	}{{
+		in:  []int{4, div, 3},
+		out: []int{3, 4},
+	}, {
+		in:  []int{4, div, 3, 3, 3},
+		out: []int{3, 3, 3, 4},
+	}, {
+		in:  []int{0, 4, div, 3},
+		out: []int{0, 3, 4},
+	}, {
+		in:  []int{0, 0, 4, 5, div, 3, 3},
+		out: []int{0, 0, 3, 3, 4, 5},
+	}, {
+		in:  []int{0, 0, 1, 4, 5, div, 3, 3},
+		out: []int{0, 0, 1, 3, 3, 4, 5},
+	}, {
+		in:  []int{0, 0, 1, 4, 5, div, 4, 4},
+		out: []int{0, 0, 1, 4, 4, 4, 5},
+	},
+	}
+	for j, tt := range tests {
+		i := Iter{}
+		var w, p int
+		for k, cc := range tt.in {
+
+			if cc == div {
+				w = 100
+				p = k
+				continue
+			}
+			i.Elems = append(i.Elems, makeCE([]int{w, defaultSecondary, 2, cc}))
+		}
+		i.doNorm(p, i.Elems[p].CCC())
+		if len(i.Elems) != len(tt.out) {
+			t.Errorf("%d: length was %d; want %d", j, len(i.Elems), len(tt.out))
+		}
+		prevCCC := uint8(0)
+		for k, ce := range i.Elems {
+			if int(ce.CCC()) != tt.out[k] {
+				t.Errorf("%d:%d: unexpected CCC. Was %d; want %d", j, k, ce.CCC(), tt.out[k])
+			}
+			if k > 0 && ce.CCC() == prevCCC && i.Elems[k-1].Primary() > ce.Primary() {
+				t.Errorf("%d:%d: normalization crossed across CCC boundary.", j, k)
+			}
+		}
+	}
+
+	// Combining rune overflow is tested in search/pattern_test.go.
+}

data/ext/vendor/golang.org/x/text/internal/colltab/numeric.go

@@ -0,0 +1,236 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"unicode"
+	"unicode/utf8"
+)
+
+// NewNumericWeighter wraps w to replace individual digits to sort based on their
+// numeric value.
+//
+// Weighter w must have a free primary weight after the primary weight for 9.
+// If this is not the case, numeric value will sort at the same primary level
+// as the first primary sorting after 9.
+func NewNumericWeighter(w Weighter) Weighter {
+	getElem := func(s string) Elem {
+		elems, _ := w.AppendNextString(nil, s)
+		return elems[0]
+	}
+	nine := getElem("9")
+
+	// Numbers should order before zero, but the DUCET has no room for this.
+	// TODO: move before zero once we use fractional collation elements.
+	ns, _ := MakeElem(nine.Primary()+1, nine.Secondary(), int(nine.Tertiary()), 0)
+
+	return &numericWeighter{
+		Weighter: w,
+
+		// We assume that w sorts digits of different kinds in order of numeric
+		// value and that the tertiary weight order is preserved.
+		//
+		// TODO: evaluate whether it is worth basing the ranges on the Elem
+		// encoding itself once the move to fractional weights is complete.
+		zero:          getElem("0"),
+		zeroSpecialLo: getElem("０"), // U+FF10 FULLWIDTH DIGIT ZERO
+		zeroSpecialHi: getElem("₀"), // U+2080 SUBSCRIPT ZERO
+		nine:          nine,
+		nineSpecialHi: getElem("₉"), // U+2089 SUBSCRIPT NINE
+		numberStart:   ns,
+	}
+}
+
+// A numericWeighter translates a stream of digits into a stream of weights
+// representing the numeric value.
+type numericWeighter struct {
+	Weighter
+
+	// The Elems below all demarcate boundaries of specific ranges. With the
+	// current element encoding digits are in two ranges: normal (default
+	// tertiary value) and special. For most languages, digits have collation
+	// elements in the normal range.
+	//
+	// Note: the range tests are very specific for the element encoding used by
+	// this implementation. The tests in collate_test.go are designed to fail
+	// if this code is not updated when an encoding has changed.
+
+	zero          Elem // normal digit zero
+	zeroSpecialLo Elem // special digit zero, low tertiary value
+	zeroSpecialHi Elem // special digit zero, high tertiary value
+	nine          Elem // normal digit nine
+	nineSpecialHi Elem // special digit nine
+	numberStart   Elem
+}
+
+// AppendNext calls the namesake of the underlying weigher, but replaces single
+// digits with weights representing their value.
+func (nw *numericWeighter) AppendNext(buf []Elem, s []byte) (ce []Elem, n int) {
+	ce, n = nw.Weighter.AppendNext(buf, s)
+	nc := numberConverter{
+		elems: buf,
+		w:     nw,
+		b:     s,
+	}
+	isZero, ok := nc.checkNextDigit(ce)
+	if !ok {
+		return ce, n
+	}
+	// ce might have been grown already, so take it instead of buf.
+	nc.init(ce, len(buf), isZero)
+	for n < len(s) {
+		ce, sz := nw.Weighter.AppendNext(nc.elems, s[n:])
+		nc.b = s
+		n += sz
+		if !nc.update(ce) {
+			break
+		}
+	}
+	return nc.result(), n
+}
+
+// AppendNextString calls the namesake of the underlying weigher, but replaces
+// single digits with weights representing their value.
+func (nw *numericWeighter) AppendNextString(buf []Elem, s string) (ce []Elem, n int) {
+	ce, n = nw.Weighter.AppendNextString(buf, s)
+	nc := numberConverter{
+		elems: buf,
+		w:     nw,
+		s:     s,
+	}
+	isZero, ok := nc.checkNextDigit(ce)
+	if !ok {
+		return ce, n
+	}
+	nc.init(ce, len(buf), isZero)
+	for n < len(s) {
+		ce, sz := nw.Weighter.AppendNextString(nc.elems, s[n:])
+		nc.s = s
+		n += sz
+		if !nc.update(ce) {
+			break
+		}
+	}
+	return nc.result(), n
+}
+
+type numberConverter struct {
+	w *numericWeighter
+
+	elems    []Elem
+	nDigits  int
+	lenIndex int
+
+	s string // set if the input was of type string
+	b []byte // set if the input was of type []byte
+}
+
+// init completes initialization of a numberConverter and prepares it for adding
+// more digits. elems is assumed to have a digit starting at oldLen.
+func (nc *numberConverter) init(elems []Elem, oldLen int, isZero bool) {
+	// Insert a marker indicating the start of a number and and a placeholder
+	// for the number of digits.
+	if isZero {
+		elems = append(elems[:oldLen], nc.w.numberStart, 0)
+	} else {
+		elems = append(elems, 0, 0)
+		copy(elems[oldLen+2:], elems[oldLen:])
+		elems[oldLen] = nc.w.numberStart
+		elems[oldLen+1] = 0
+
+		nc.nDigits = 1
+	}
+	nc.elems = elems
+	nc.lenIndex = oldLen + 1
+}
+
+// checkNextDigit reports whether bufNew adds a single digit relative to the old
+// buffer. If it does, it also reports whether this digit is zero.
+func (nc *numberConverter) checkNextDigit(bufNew []Elem) (isZero, ok bool) {
+	if len(nc.elems) >= len(bufNew) {
+		return false, false
+	}
+	e := bufNew[len(nc.elems)]
+	if e < nc.w.zeroSpecialLo || nc.w.nine < e {
+		// Not a number.
+		return false, false
+	}
+	if e < nc.w.zero {
+		if e > nc.w.nineSpecialHi {
+			// Not a number.
+			return false, false
+		}
+		if !nc.isDigit() {
+			return false, false
+		}
+		isZero = e <= nc.w.zeroSpecialHi
+	} else {
+		// This is the common case if we encounter a digit.
+		isZero = e == nc.w.zero
+	}
+	// Test the remaining added collation elements have a zero primary value.
+	if n := len(bufNew) - len(nc.elems); n > 1 {
+		for i := len(nc.elems) + 1; i < len(bufNew); i++ {
+			if bufNew[i].Primary() != 0 {
+				return false, false
+			}
+		}
+		// In some rare cases, collation elements will encode runes in
+		// unicode.No as a digit. For example Ethiopic digits (U+1369 - U+1371)
+		// are not in Nd. Also some digits that clearly belong in unicode.No,
+		// like U+0C78 TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR, have
+		// collation elements indistinguishable from normal digits.
+		// Unfortunately, this means we need to make this check for nearly all
+		// non-Latin digits.
+		//
+		// TODO: check the performance impact and find something better if it is
+		// an issue.
+		if !nc.isDigit() {
+			return false, false
+		}
+	}
+	return isZero, true
+}
+
+func (nc *numberConverter) isDigit() bool {
+	if nc.b != nil {
+		r, _ := utf8.DecodeRune(nc.b)
+		return unicode.In(r, unicode.Nd)
+	}
+	r, _ := utf8.DecodeRuneInString(nc.s)
+	return unicode.In(r, unicode.Nd)
+}
+
+// We currently support a maximum of about 2M digits (the number of primary
+// values). Such numbers will compare correctly against small numbers, but their
+// comparison against other large numbers is undefined.
+//
+// TODO: define a proper fallback, such as comparing large numbers textually or
+// actually allowing numbers of unlimited length.
+//
+// TODO: cap this to a lower number (like 100) and maybe allow a larger number
+// in an option?
+const maxDigits = 1<<maxPrimaryBits - 1
+
+func (nc *numberConverter) update(elems []Elem) bool {
+	isZero, ok := nc.checkNextDigit(elems)
+	if nc.nDigits == 0 && isZero {
+		return true
+	}
+	nc.elems = elems
+	if !ok {
+		return false
+	}
+	nc.nDigits++
+	return nc.nDigits < maxDigits
+}
+
+// result fills in the length element for the digit sequence and returns the
+// completed collation elements.
+func (nc *numberConverter) result() []Elem {
+	e, _ := MakeElem(nc.nDigits, defaultSecondary, defaultTertiary, 0)
+	nc.elems[nc.lenIndex] = e
+	return nc.elems
+}