ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/internal/colltab/numeric_test.go

@@ -0,0 +1,159 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"golang.org/x/text/internal/testtext"
+)
+
+const (
+	digSec  = defaultSecondary
+	digTert = defaultTertiary
+)
+
+var tPlus3 = e(0, 50, digTert+3)
+
+// numWeighter is a testWeighter used for testing numericWeighter.
+var numWeighter = testWeighter{
+	"0": p(100),
+	"０": []Elem{e(100, digSec, digTert+1)}, // U+FF10 FULLWIDTH DIGIT ZERO
+	"₀": []Elem{e(100, digSec, digTert+5)}, // U+2080 SUBSCRIPT ZERO
+
+	"1": p(101),
+	// Allow non-primary collation elements to be inserted.
+	"١": append(p(101), tPlus3), // U+0661 ARABIC-INDIC DIGIT ONE
+	// Allow varying tertiary weight if the number is Nd.
+	"１": []Elem{e(101, digSec, digTert+1)}, // U+FF11 FULLWIDTH DIGIT ONE
+	"2": p(102),
+	// Allow non-primary collation elements to be inserted.
+	"٢": append(p(102), tPlus3), // U+0662 ARABIC-INDIC DIGIT TWO
+	// Varying tertiary weights should be ignored.
+	"２": []Elem{e(102, digSec, digTert+3)}, // U+FF12 FULLWIDTH DIGIT TWO
+	"3": p(103),
+	"4": p(104),
+	"5": p(105),
+	"6": p(106),
+	"7": p(107),
+	// Weights must be strictly monotonically increasing, but do not need to be
+	// consecutive.
+	"8": p(118),
+	"9": p(119),
+	// Allow non-primary collation elements to be inserted.
+	"٩": append(p(119), tPlus3), // U+0669 ARABIC-INDIC DIGIT NINE
+	// Varying tertiary weights should be ignored.
+	"９": []Elem{e(119, digSec, digTert+1)}, // U+FF19 FULLWIDTH DIGIT NINE
+	"₉": []Elem{e(119, digSec, digTert+5)}, // U+2089 SUBSCRIPT NINE
+
+	"a": p(5),
+	"b": p(6),
+	"c": p(8, 2),
+
+	"klm": p(99),
+
+	"nop": p(121),
+
+	"x": p(200),
+	"y": p(201),
+}
+
+func p(w ...int) (elems []Elem) {
+	for _, x := range w {
+		e, _ := MakeElem(x, digSec, digTert, 0)
+		elems = append(elems, e)
+	}
+	return elems
+}
+
+func TestNumericAppendNext(t *testing.T) {
+	for _, tt := range []struct {
+		in string
+		w  []Elem
+	}{
+		{"a", p(5)},
+		{"klm", p(99)},
+		{"aa", p(5, 5)},
+		{"1", p(120, 1, 101)},
+		{"0", p(120, 0)},
+		{"01", p(120, 1, 101)},
+		{"0001", p(120, 1, 101)},
+		{"10", p(120, 2, 101, 100)},
+		{"99", p(120, 2, 119, 119)},
+		{"9999", p(120, 4, 119, 119, 119, 119)},
+		{"1a", p(120, 1, 101, 5)},
+		{"0b", p(120, 0, 6)},
+		{"01c", p(120, 1, 101, 8, 2)},
+		{"10x", p(120, 2, 101, 100, 200)},
+		{"99y", p(120, 2, 119, 119, 201)},
+		{"9999nop", p(120, 4, 119, 119, 119, 119, 121)},
+
+		// Allow follow-up collation elements if they have a zero non-primary.
+		{"١٢٩", []Elem{e(120), e(3), e(101), tPlus3, e(102), tPlus3, e(119), tPlus3}},
+		{
+			"１２９",
+			[]Elem{
+				e(120), e(3),
+				e(101, digSec, digTert+1),
+				e(102, digSec, digTert+3),
+				e(119, digSec, digTert+1),
+			},
+		},
+
+		// Ensure AppendNext* adds to the given buffer.
+		{"a10", p(5, 120, 2, 101, 100)},
+	} {
+		nw := NewNumericWeighter(numWeighter)
+
+		b := []byte(tt.in)
+		got := []Elem(nil)
+		for n, sz := 0, 0; n < len(b); {
+			got, sz = nw.AppendNext(got, b[n:])
+			n += sz
+		}
+		if !reflect.DeepEqual(got, tt.w) {
+			t.Errorf("AppendNext(%q) =\n%v; want\n%v", tt.in, got, tt.w)
+		}
+
+		got = nil
+		for n, sz := 0, 0; n < len(tt.in); {
+			got, sz = nw.AppendNextString(got, tt.in[n:])
+			n += sz
+		}
+		if !reflect.DeepEqual(got, tt.w) {
+			t.Errorf("AppendNextString(%q) =\n%v; want\n%v", tt.in, got, tt.w)
+		}
+	}
+}
+
+func TestNumericOverflow(t *testing.T) {
+	manyDigits := strings.Repeat("9", maxDigits+1) + "a"
+
+	nw := NewNumericWeighter(numWeighter)
+
+	got, n := nw.AppendNextString(nil, manyDigits)
+
+	if n != maxDigits {
+		t.Errorf("n: got %d; want %d", n, maxDigits)
+	}
+
+	if got[1].Primary() != maxDigits {
+		t.Errorf("primary(e[1]): got %d; want %d", n, maxDigits)
+	}
+}
+
+func TestNumericWeighterAlloc(t *testing.T) {
+	buf := make([]Elem, 100)
+	w := NewNumericWeighter(numWeighter)
+	s := "1234567890a"
+
+	nNormal := testtext.AllocsPerRun(3, func() { numWeighter.AppendNextString(buf, s) })
+	nNumeric := testtext.AllocsPerRun(3, func() { w.AppendNextString(buf, s) })
+	if n := nNumeric - nNormal; n > 0 {
+		t.Errorf("got %f; want 0", n)
+	}
+}

data/ext/vendor/golang.org/x/text/internal/colltab/table.go

@@ -0,0 +1,275 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/unicode/norm"
+)
+
+// Table holds all collation data for a given collation ordering.
+type Table struct {
+	Index Trie // main trie
+
+	// expansion info
+	ExpandElem []uint32
+
+	// contraction info
+	ContractTries  ContractTrieSet
+	ContractElem   []uint32
+	MaxContractLen int
+	VariableTop    uint32
+}
+
+func (t *Table) AppendNext(w []Elem, b []byte) (res []Elem, n int) {
+	return t.appendNext(w, source{bytes: b})
+}
+
+func (t *Table) AppendNextString(w []Elem, s string) (res []Elem, n int) {
+	return t.appendNext(w, source{str: s})
+}
+
+func (t *Table) Start(p int, b []byte) int {
+	// TODO: implement
+	panic("not implemented")
+}
+
+func (t *Table) StartString(p int, s string) int {
+	// TODO: implement
+	panic("not implemented")
+}
+
+func (t *Table) Domain() []string {
+	// TODO: implement
+	panic("not implemented")
+}
+
+func (t *Table) Top() uint32 {
+	return t.VariableTop
+}
+
+type source struct {
+	str   string
+	bytes []byte
+}
+
+func (src *source) lookup(t *Table) (ce Elem, sz int) {
+	if src.bytes == nil {
+		return t.Index.lookupString(src.str)
+	}
+	return t.Index.lookup(src.bytes)
+}
+
+func (src *source) tail(sz int) {
+	if src.bytes == nil {
+		src.str = src.str[sz:]
+	} else {
+		src.bytes = src.bytes[sz:]
+	}
+}
+
+func (src *source) nfd(buf []byte, end int) []byte {
+	if src.bytes == nil {
+		return norm.NFD.AppendString(buf[:0], src.str[:end])
+	}
+	return norm.NFD.Append(buf[:0], src.bytes[:end]...)
+}
+
+func (src *source) rune() (r rune, sz int) {
+	if src.bytes == nil {
+		return utf8.DecodeRuneInString(src.str)
+	}
+	return utf8.DecodeRune(src.bytes)
+}
+
+func (src *source) properties(f norm.Form) norm.Properties {
+	if src.bytes == nil {
+		return f.PropertiesString(src.str)
+	}
+	return f.Properties(src.bytes)
+}
+
+// appendNext appends the weights corresponding to the next rune or
+// contraction in s.  If a contraction is matched to a discontinuous
+// sequence of runes, the weights for the interstitial runes are
+// appended as well.  It returns a new slice that includes the appended
+// weights and the number of bytes consumed from s.
+func (t *Table) appendNext(w []Elem, src source) (res []Elem, n int) {
+	ce, sz := src.lookup(t)
+	tp := ce.ctype()
+	if tp == ceNormal {
+		if ce == 0 {
+			r, _ := src.rune()
+			const (
+				hangulSize  = 3
+				firstHangul = 0xAC00
+				lastHangul  = 0xD7A3
+			)
+			if r >= firstHangul && r <= lastHangul {
+				// TODO: performance can be considerably improved here.
+				n = sz
+				var buf [16]byte // Used for decomposing Hangul.
+				for b := src.nfd(buf[:0], hangulSize); len(b) > 0; b = b[sz:] {
+					ce, sz = t.Index.lookup(b)
+					w = append(w, ce)
+				}
+				return w, n
+			}
+			ce = makeImplicitCE(implicitPrimary(r))
+		}
+		w = append(w, ce)
+	} else if tp == ceExpansionIndex {
+		w = t.appendExpansion(w, ce)
+	} else if tp == ceContractionIndex {
+		n := 0
+		src.tail(sz)
+		if src.bytes == nil {
+			w, n = t.matchContractionString(w, ce, src.str)
+		} else {
+			w, n = t.matchContraction(w, ce, src.bytes)
+		}
+		sz += n
+	} else if tp == ceDecompose {
+		// Decompose using NFKD and replace tertiary weights.
+		t1, t2 := splitDecompose(ce)
+		i := len(w)
+		nfkd := src.properties(norm.NFKD).Decomposition()
+		for p := 0; len(nfkd) > 0; nfkd = nfkd[p:] {
+			w, p = t.appendNext(w, source{bytes: nfkd})
+		}
+		w[i] = w[i].updateTertiary(t1)
+		if i++; i < len(w) {
+			w[i] = w[i].updateTertiary(t2)
+			for i++; i < len(w); i++ {
+				w[i] = w[i].updateTertiary(maxTertiary)
+			}
+		}
+	}
+	return w, sz
+}
+
+func (t *Table) appendExpansion(w []Elem, ce Elem) []Elem {
+	i := splitExpandIndex(ce)
+	n := int(t.ExpandElem[i])
+	i++
+	for _, ce := range t.ExpandElem[i : i+n] {
+		w = append(w, Elem(ce))
+	}
+	return w
+}
+
+func (t *Table) matchContraction(w []Elem, ce Elem, suffix []byte) ([]Elem, int) {
+	index, n, offset := splitContractIndex(ce)
+
+	scan := t.ContractTries.scanner(index, n, suffix)
+	buf := [norm.MaxSegmentSize]byte{}
+	bufp := 0
+	p := scan.scan(0)
+
+	if !scan.done && p < len(suffix) && suffix[p] >= utf8.RuneSelf {
+		// By now we should have filtered most cases.
+		p0 := p
+		bufn := 0
+		rune := norm.NFD.Properties(suffix[p:])
+		p += rune.Size()
+		if rune.LeadCCC() != 0 {
+			prevCC := rune.TrailCCC()
+			// A gap may only occur in the last normalization segment.
+			// This also ensures that len(scan.s) < norm.MaxSegmentSize.
+			if end := norm.NFD.FirstBoundary(suffix[p:]); end != -1 {
+				scan.s = suffix[:p+end]
+			}
+			for p < len(suffix) && !scan.done && suffix[p] >= utf8.RuneSelf {
+				rune = norm.NFD.Properties(suffix[p:])
+				if ccc := rune.LeadCCC(); ccc == 0 || prevCC >= ccc {
+					break
+				}
+				prevCC = rune.TrailCCC()
+				if pp := scan.scan(p); pp != p {
+					// Copy the interstitial runes for later processing.
+					bufn += copy(buf[bufn:], suffix[p0:p])
+					if scan.pindex == pp {
+						bufp = bufn
+					}
+					p, p0 = pp, pp
+				} else {
+					p += rune.Size()
+				}
+			}
+		}
+	}
+	// Append weights for the matched contraction, which may be an expansion.
+	i, n := scan.result()
+	ce = Elem(t.ContractElem[i+offset])
+	if ce.ctype() == ceNormal {
+		w = append(w, ce)
+	} else {
+		w = t.appendExpansion(w, ce)
+	}
+	// Append weights for the runes in the segment not part of the contraction.
+	for b, p := buf[:bufp], 0; len(b) > 0; b = b[p:] {
+		w, p = t.appendNext(w, source{bytes: b})
+	}
+	return w, n
+}
+
+// TODO: unify the two implementations. This is best done after first simplifying
+// the algorithm taking into account the inclusion of both NFC and NFD forms
+// in the table.
+func (t *Table) matchContractionString(w []Elem, ce Elem, suffix string) ([]Elem, int) {
+	index, n, offset := splitContractIndex(ce)
+
+	scan := t.ContractTries.scannerString(index, n, suffix)
+	buf := [norm.MaxSegmentSize]byte{}
+	bufp := 0
+	p := scan.scan(0)
+
+	if !scan.done && p < len(suffix) && suffix[p] >= utf8.RuneSelf {
+		// By now we should have filtered most cases.
+		p0 := p
+		bufn := 0
+		rune := norm.NFD.PropertiesString(suffix[p:])
+		p += rune.Size()
+		if rune.LeadCCC() != 0 {
+			prevCC := rune.TrailCCC()
+			// A gap may only occur in the last normalization segment.
+			// This also ensures that len(scan.s) < norm.MaxSegmentSize.
+			if end := norm.NFD.FirstBoundaryInString(suffix[p:]); end != -1 {
+				scan.s = suffix[:p+end]
+			}
+			for p < len(suffix) && !scan.done && suffix[p] >= utf8.RuneSelf {
+				rune = norm.NFD.PropertiesString(suffix[p:])
+				if ccc := rune.LeadCCC(); ccc == 0 || prevCC >= ccc {
+					break
+				}
+				prevCC = rune.TrailCCC()
+				if pp := scan.scan(p); pp != p {
+					// Copy the interstitial runes for later processing.
+					bufn += copy(buf[bufn:], suffix[p0:p])
+					if scan.pindex == pp {
+						bufp = bufn
+					}
+					p, p0 = pp, pp
+				} else {
+					p += rune.Size()
+				}
+			}
+		}
+	}
+	// Append weights for the matched contraction, which may be an expansion.
+	i, n := scan.result()
+	ce = Elem(t.ContractElem[i+offset])
+	if ce.ctype() == ceNormal {
+		w = append(w, ce)
+	} else {
+		w = t.appendExpansion(w, ce)
+	}
+	// Append weights for the runes in the segment not part of the contraction.
+	for b, p := buf[:bufp], 0; len(b) > 0; b = b[p:] {
+		w, p = t.appendNext(w, source{bytes: b})
+	}
+	return w, n
+}

data/ext/vendor/golang.org/x/text/internal/colltab/trie.go

@@ -0,0 +1,159 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The trie in this file is used to associate the first full character in an
+// UTF-8 string to a collation element. All but the last byte in a UTF-8 byte
+// sequence are used to lookup offsets in the index table to be used for the
+// next byte. The last byte is used to index into a table of collation elements.
+// For a full description, see go.text/collate/build/trie.go.
+
+package colltab
+
+const blockSize = 64
+
+type Trie struct {
+	Index0  []uint16 // index for first byte (0xC0-0xFF)
+	Values0 []uint32 // index for first byte (0x00-0x7F)
+	Index   []uint16
+	Values  []uint32
+}
+
+const (
+	t1 = 0x00 // 0000 0000
+	tx = 0x80 // 1000 0000
+	t2 = 0xC0 // 1100 0000
+	t3 = 0xE0 // 1110 0000
+	t4 = 0xF0 // 1111 0000
+	t5 = 0xF8 // 1111 1000
+	t6 = 0xFC // 1111 1100
+	te = 0xFE // 1111 1110
+)
+
+func (t *Trie) lookupValue(n uint16, b byte) Elem {
+	return Elem(t.Values[int(n)<<6+int(b)])
+}
+
+// lookup returns the trie value for the first UTF-8 encoding in s and
+// the width in bytes of this encoding. The size will be 0 if s does not
+// hold enough bytes to complete the encoding. len(s) must be greater than 0.
+func (t *Trie) lookup(s []byte) (v Elem, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < tx:
+		return Elem(t.Values0[c0]), 1
+	case c0 < t2:
+		return 0, 1
+	case c0 < t3:
+		if len(s) < 2 {
+			return 0, 0
+		}
+		i := t.Index0[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		return t.lookupValue(i, c1), 2
+	case c0 < t4:
+		if len(s) < 3 {
+			return 0, 0
+		}
+		i := t.Index0[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := int(i)<<6 + int(c1)
+		i = t.Index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		return t.lookupValue(i, c2), 3
+	case c0 < t5:
+		if len(s) < 4 {
+			return 0, 0
+		}
+		i := t.Index0[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := int(i)<<6 + int(c1)
+		i = t.Index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		o = int(i)<<6 + int(c2)
+		i = t.Index[o]
+		c3 := s[3]
+		if c3 < tx || t2 <= c3 {
+			return 0, 3
+		}
+		return t.lookupValue(i, c3), 4
+	}
+	// Illegal rune
+	return 0, 1
+}
+
+// The body of lookupString is a verbatim copy of that of lookup.
+func (t *Trie) lookupString(s string) (v Elem, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < tx:
+		return Elem(t.Values0[c0]), 1
+	case c0 < t2:
+		return 0, 1
+	case c0 < t3:
+		if len(s) < 2 {
+			return 0, 0
+		}
+		i := t.Index0[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		return t.lookupValue(i, c1), 2
+	case c0 < t4:
+		if len(s) < 3 {
+			return 0, 0
+		}
+		i := t.Index0[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := int(i)<<6 + int(c1)
+		i = t.Index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		return t.lookupValue(i, c2), 3
+	case c0 < t5:
+		if len(s) < 4 {
+			return 0, 0
+		}
+		i := t.Index0[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := int(i)<<6 + int(c1)
+		i = t.Index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		o = int(i)<<6 + int(c2)
+		i = t.Index[o]
+		c3 := s[3]
+		if c3 < tx || t2 <= c3 {
+			return 0, 3
+		}
+		return t.lookupValue(i, c3), 4
+	}
+	// Illegal rune
+	return 0, 1
+}