ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/collate/build/colelem_test.go

@@ -0,0 +1,215 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	"testing"
+
+	"golang.org/x/text/internal/colltab"
+)
+
+type ceTest struct {
+	f   func(in []int) (uint32, error)
+	arg []int
+	val uint32
+}
+
+func normalCE(in []int) (ce uint32, err error) {
+	return makeCE(rawCE{w: in[:3], ccc: uint8(in[3])})
+}
+
+func expandCE(in []int) (ce uint32, err error) {
+	return makeExpandIndex(in[0])
+}
+
+func contractCE(in []int) (ce uint32, err error) {
+	return makeContractIndex(ctHandle{in[0], in[1]}, in[2])
+}
+
+func decompCE(in []int) (ce uint32, err error) {
+	return makeDecompose(in[0], in[1])
+}
+
+var ceTests = []ceTest{
+	{normalCE, []int{0, 0, 0, 0}, 0xA0000000},
+	{normalCE, []int{0, 0x28, 3, 0}, 0xA0002803},
+	{normalCE, []int{0, 0x28, 3, 0xFF}, 0xAFF02803},
+	{normalCE, []int{100, defaultSecondary, 3, 0}, 0x0000C883},
+	// non-ignorable primary with non-default secondary
+	{normalCE, []int{100, 0x28, defaultTertiary, 0}, 0x4000C828},
+	{normalCE, []int{100, defaultSecondary + 8, 3, 0}, 0x0000C983},
+	{normalCE, []int{100, 0, 3, 0}, 0xFFFF}, // non-ignorable primary with non-supported secondary
+	{normalCE, []int{100, 1, 3, 0}, 0xFFFF},
+	{normalCE, []int{1 << maxPrimaryBits, defaultSecondary, 0, 0}, 0xFFFF},
+	{normalCE, []int{0, 1 << maxSecondaryBits, 0, 0}, 0xFFFF},
+	{normalCE, []int{100, defaultSecondary, 1 << maxTertiaryBits, 0}, 0xFFFF},
+	{normalCE, []int{0x123, defaultSecondary, 8, 0xFF}, 0x88FF0123},
+	{normalCE, []int{0x123, defaultSecondary + 1, 8, 0xFF}, 0xFFFF},
+
+	{contractCE, []int{0, 0, 0}, 0xC0000000},
+	{contractCE, []int{1, 1, 1}, 0xC0010011},
+	{contractCE, []int{1, (1 << maxNBits) - 1, 1}, 0xC001001F},
+	{contractCE, []int{(1 << maxTrieIndexBits) - 1, 1, 1}, 0xC001FFF1},
+	{contractCE, []int{1, 1, (1 << maxContractOffsetBits) - 1}, 0xDFFF0011},
+	{contractCE, []int{1, (1 << maxNBits), 1}, 0xFFFF},
+	{contractCE, []int{(1 << maxTrieIndexBits), 1, 1}, 0xFFFF},
+	{contractCE, []int{1, (1 << maxContractOffsetBits), 1}, 0xFFFF},
+
+	{expandCE, []int{0}, 0xE0000000},
+	{expandCE, []int{5}, 0xE0000005},
+	{expandCE, []int{(1 << maxExpandIndexBits) - 1}, 0xE000FFFF},
+	{expandCE, []int{1 << maxExpandIndexBits}, 0xFFFF},
+
+	{decompCE, []int{0, 0}, 0xF0000000},
+	{decompCE, []int{1, 1}, 0xF0000101},
+	{decompCE, []int{0x1F, 0x1F}, 0xF0001F1F},
+	{decompCE, []int{256, 0x1F}, 0xFFFF},
+	{decompCE, []int{0x1F, 256}, 0xFFFF},
+}
+
+func TestColElem(t *testing.T) {
+	for i, tt := range ceTests {
+		in := make([]int, len(tt.arg))
+		copy(in, tt.arg)
+		ce, err := tt.f(in)
+		if tt.val == 0xFFFF {
+			if err == nil {
+				t.Errorf("%d: expected error for args %x", i, tt.arg)
+			}
+			continue
+		}
+		if err != nil {
+			t.Errorf("%d: unexpected error: %v", i, err.Error())
+		}
+		if ce != tt.val {
+			t.Errorf("%d: colElem=%X; want %X", i, ce, tt.val)
+		}
+	}
+}
+
+func mkRawCES(in [][]int) []rawCE {
+	out := []rawCE{}
+	for _, w := range in {
+		out = append(out, rawCE{w: w})
+	}
+	return out
+}
+
+type weightsTest struct {
+	a, b   [][]int
+	level  colltab.Level
+	result int
+}
+
+var nextWeightTests = []weightsTest{
+	{
+		a:     [][]int{{100, 20, 5, 0}},
+		b:     [][]int{{101, defaultSecondary, defaultTertiary, 0}},
+		level: colltab.Primary,
+	},
+	{
+		a:     [][]int{{100, 20, 5, 0}},
+		b:     [][]int{{100, 21, defaultTertiary, 0}},
+		level: colltab.Secondary,
+	},
+	{
+		a:     [][]int{{100, 20, 5, 0}},
+		b:     [][]int{{100, 20, 6, 0}},
+		level: colltab.Tertiary,
+	},
+	{
+		a:     [][]int{{100, 20, 5, 0}},
+		b:     [][]int{{100, 20, 5, 0}},
+		level: colltab.Identity,
+	},
+}
+
+var extra = [][]int{{200, 32, 8, 0}, {0, 32, 8, 0}, {0, 0, 8, 0}, {0, 0, 0, 0}}
+
+func TestNextWeight(t *testing.T) {
+	for i, tt := range nextWeightTests {
+		test := func(l colltab.Level, tt weightsTest, a, gold [][]int) {
+			res := nextWeight(tt.level, mkRawCES(a))
+			if !equalCEArrays(mkRawCES(gold), res) {
+				t.Errorf("%d:%d: expected weights %d; found %d", i, l, gold, res)
+			}
+		}
+		test(-1, tt, tt.a, tt.b)
+		for l := colltab.Primary; l <= colltab.Tertiary; l++ {
+			if tt.level <= l {
+				test(l, tt, append(tt.a, extra[l]), tt.b)
+			} else {
+				test(l, tt, append(tt.a, extra[l]), append(tt.b, extra[l]))
+			}
+		}
+	}
+}
+
+var compareTests = []weightsTest{
+	{
+		[][]int{{100, 20, 5, 0}},
+		[][]int{{100, 20, 5, 0}},
+		colltab.Identity,
+		0,
+	},
+	{
+		[][]int{{100, 20, 5, 0}, extra[0]},
+		[][]int{{100, 20, 5, 1}},
+		colltab.Primary,
+		1,
+	},
+	{
+		[][]int{{100, 20, 5, 0}},
+		[][]int{{101, 20, 5, 0}},
+		colltab.Primary,
+		-1,
+	},
+	{
+		[][]int{{101, 20, 5, 0}},
+		[][]int{{100, 20, 5, 0}},
+		colltab.Primary,
+		1,
+	},
+	{
+		[][]int{{100, 0, 0, 0}, {0, 20, 5, 0}},
+		[][]int{{0, 20, 5, 0}, {100, 0, 0, 0}},
+		colltab.Identity,
+		0,
+	},
+	{
+		[][]int{{100, 20, 5, 0}},
+		[][]int{{100, 21, 5, 0}},
+		colltab.Secondary,
+		-1,
+	},
+	{
+		[][]int{{100, 20, 5, 0}},
+		[][]int{{100, 20, 2, 0}},
+		colltab.Tertiary,
+		1,
+	},
+	{
+		[][]int{{100, 20, 5, 1}},
+		[][]int{{100, 20, 5, 2}},
+		colltab.Quaternary,
+		-1,
+	},
+}
+
+func TestCompareWeights(t *testing.T) {
+	for i, tt := range compareTests {
+		test := func(tt weightsTest, a, b [][]int) {
+			res, level := compareWeights(mkRawCES(a), mkRawCES(b))
+			if res != tt.result {
+				t.Errorf("%d: expected comparison result %d; found %d", i, tt.result, res)
+			}
+			if level != tt.level {
+				t.Errorf("%d: expected level %d; found %d", i, tt.level, level)
+			}
+		}
+		test(tt, tt.a, tt.b)
+		test(tt, append(tt.a, extra[0]), append(tt.b, extra[0]))
+	}
+}

data/ext/vendor/golang.org/x/text/collate/build/contract.go

@@ -0,0 +1,309 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"sort"
+	"strings"
+
+	"golang.org/x/text/internal/colltab"
+)
+
+// This file contains code for detecting contractions and generating
+// the necessary tables.
+// Any Unicode Collation Algorithm (UCA) table entry that has more than
+// one rune one the left-hand side is called a contraction.
+// See http://www.unicode.org/reports/tr10/#Contractions for more details.
+//
+// We define the following terms:
+//   initial:     a rune that appears as the first rune in a contraction.
+//   suffix:      a sequence of runes succeeding the initial rune
+//                in a given contraction.
+//   non-initial: a rune that appears in a suffix.
+//
+// A rune may be both an initial and a non-initial and may be so in
+// many contractions.  An initial may typically also appear by itself.
+// In case of ambiguities, the UCA requires we match the longest
+// contraction.
+//
+// Many contraction rules share the same set of possible suffixes.
+// We store sets of suffixes in a trie that associates an index with
+// each suffix in the set.  This index can be used to look up a
+// collation element associated with the (starter rune, suffix) pair.
+//
+// The trie is defined on a UTF-8 byte sequence.
+// The overall trie is represented as an array of ctEntries.  Each node of the trie
+// is represented as a subsequence of ctEntries, where each entry corresponds to
+// a possible match of a next character in the search string.  An entry
+// also includes the length and offset to the next sequence of entries
+// to check in case of a match.
+
+const (
+	final   = 0
+	noIndex = 0xFF
+)
+
+// ctEntry associates to a matching byte an offset and/or next sequence of
+// bytes to check. A ctEntry c is called final if a match means that the
+// longest suffix has been found.  An entry c is final if c.N == 0.
+// A single final entry can match a range of characters to an offset.
+// A non-final entry always matches a single byte. Note that a non-final
+// entry might still resemble a completed suffix.
+// Examples:
+// The suffix strings "ab" and "ac" can be represented as:
+// []ctEntry{
+//     {'a', 1, 1, noIndex},  // 'a' by itself does not match, so i is 0xFF.
+//     {'b', 'c', 0, 1},   // "ab" -> 1, "ac" -> 2
+// }
+//
+// The suffix strings "ab", "abc", "abd", and "abcd" can be represented as:
+// []ctEntry{
+//     {'a', 1, 1, noIndex}, // 'a' must be followed by 'b'.
+//     {'b', 1, 2, 1},    // "ab" -> 1, may be followed by 'c' or 'd'.
+//     {'d', 'd', final, 3},  // "abd" -> 3
+//     {'c', 4, 1, 2},    // "abc" -> 2, may be followed by 'd'.
+//     {'d', 'd', final, 4},  // "abcd" -> 4
+// }
+// See genStateTests in contract_test.go for more examples.
+type ctEntry struct {
+	L uint8 // non-final: byte value to match; final: lowest match in range.
+	H uint8 // non-final: relative index to next block; final: highest match in range.
+	N uint8 // non-final: length of next block; final: final
+	I uint8 // result offset. Will be noIndex if more bytes are needed to complete.
+}
+
+// contractTrieSet holds a set of contraction tries. The tries are stored
+// consecutively in the entry field.
+type contractTrieSet []struct{ l, h, n, i uint8 }
+
+// ctHandle is used to identify a trie in the trie set, consisting in an offset
+// in the array and the size of the first node.
+type ctHandle struct {
+	index, n int
+}
+
+// appendTrie adds a new trie for the given suffixes to the trie set and returns
+// a handle to it.  The handle will be invalid on error.
+func appendTrie(ct *colltab.ContractTrieSet, suffixes []string) (ctHandle, error) {
+	es := make([]stridx, len(suffixes))
+	for i, s := range suffixes {
+		es[i].str = s
+	}
+	sort.Sort(offsetSort(es))
+	for i := range es {
+		es[i].index = i + 1
+	}
+	sort.Sort(genidxSort(es))
+	i := len(*ct)
+	n, err := genStates(ct, es)
+	if err != nil {
+		*ct = (*ct)[:i]
+		return ctHandle{}, err
+	}
+	return ctHandle{i, n}, nil
+}
+
+// genStates generates ctEntries for a given suffix set and returns
+// the number of entries for the first node.
+func genStates(ct *colltab.ContractTrieSet, sis []stridx) (int, error) {
+	if len(sis) == 0 {
+		return 0, fmt.Errorf("genStates: list of suffices must be non-empty")
+	}
+	start := len(*ct)
+	// create entries for differing first bytes.
+	for _, si := range sis {
+		s := si.str
+		if len(s) == 0 {
+			continue
+		}
+		added := false
+		c := s[0]
+		if len(s) > 1 {
+			for j := len(*ct) - 1; j >= start; j-- {
+				if (*ct)[j].L == c {
+					added = true
+					break
+				}
+			}
+			if !added {
+				*ct = append(*ct, ctEntry{L: c, I: noIndex})
+			}
+		} else {
+			for j := len(*ct) - 1; j >= start; j-- {
+				// Update the offset for longer suffixes with the same byte.
+				if (*ct)[j].L == c {
+					(*ct)[j].I = uint8(si.index)
+					added = true
+				}
+				// Extend range of final ctEntry, if possible.
+				if (*ct)[j].H+1 == c {
+					(*ct)[j].H = c
+					added = true
+				}
+			}
+			if !added {
+				*ct = append(*ct, ctEntry{L: c, H: c, N: final, I: uint8(si.index)})
+			}
+		}
+	}
+	n := len(*ct) - start
+	// Append nodes for the remainder of the suffixes for each ctEntry.
+	sp := 0
+	for i, end := start, len(*ct); i < end; i++ {
+		fe := (*ct)[i]
+		if fe.H == 0 { // uninitialized non-final
+			ln := len(*ct) - start - n
+			if ln > 0xFF {
+				return 0, fmt.Errorf("genStates: relative block offset too large: %d > 255", ln)
+			}
+			fe.H = uint8(ln)
+			// Find first non-final strings with same byte as current entry.
+			for ; sis[sp].str[0] != fe.L; sp++ {
+			}
+			se := sp + 1
+			for ; se < len(sis) && len(sis[se].str) > 1 && sis[se].str[0] == fe.L; se++ {
+			}
+			sl := sis[sp:se]
+			sp = se
+			for i, si := range sl {
+				sl[i].str = si.str[1:]
+			}
+			nn, err := genStates(ct, sl)
+			if err != nil {
+				return 0, err
+			}
+			fe.N = uint8(nn)
+			(*ct)[i] = fe
+		}
+	}
+	sort.Sort(entrySort((*ct)[start : start+n]))
+	return n, nil
+}
+
+// There may be both a final and non-final entry for a byte if the byte
+// is implied in a range of matches in the final entry.
+// We need to ensure that the non-final entry comes first in that case.
+type entrySort colltab.ContractTrieSet
+
+func (fe entrySort) Len() int      { return len(fe) }
+func (fe entrySort) Swap(i, j int) { fe[i], fe[j] = fe[j], fe[i] }
+func (fe entrySort) Less(i, j int) bool {
+	return fe[i].L > fe[j].L
+}
+
+// stridx is used for sorting suffixes and their associated offsets.
+type stridx struct {
+	str   string
+	index int
+}
+
+// For computing the offsets, we first sort by size, and then by string.
+// This ensures that strings that only differ in the last byte by 1
+// are sorted consecutively in increasing order such that they can
+// be packed as a range in a final ctEntry.
+type offsetSort []stridx
+
+func (si offsetSort) Len() int      { return len(si) }
+func (si offsetSort) Swap(i, j int) { si[i], si[j] = si[j], si[i] }
+func (si offsetSort) Less(i, j int) bool {
+	if len(si[i].str) != len(si[j].str) {
+		return len(si[i].str) > len(si[j].str)
+	}
+	return si[i].str < si[j].str
+}
+
+// For indexing, we want to ensure that strings are sorted in string order, where
+// for strings with the same prefix, we put longer strings before shorter ones.
+type genidxSort []stridx
+
+func (si genidxSort) Len() int      { return len(si) }
+func (si genidxSort) Swap(i, j int) { si[i], si[j] = si[j], si[i] }
+func (si genidxSort) Less(i, j int) bool {
+	if strings.HasPrefix(si[j].str, si[i].str) {
+		return false
+	}
+	if strings.HasPrefix(si[i].str, si[j].str) {
+		return true
+	}
+	return si[i].str < si[j].str
+}
+
+// lookup matches the longest suffix in str and returns the associated offset
+// and the number of bytes consumed.
+func lookup(ct *colltab.ContractTrieSet, h ctHandle, str []byte) (index, ns int) {
+	states := (*ct)[h.index:]
+	p := 0
+	n := h.n
+	for i := 0; i < n && p < len(str); {
+		e := states[i]
+		c := str[p]
+		if c >= e.L {
+			if e.L == c {
+				p++
+				if e.I != noIndex {
+					index, ns = int(e.I), p
+				}
+				if e.N != final {
+					// set to new state
+					i, states, n = 0, states[int(e.H)+n:], int(e.N)
+				} else {
+					return
+				}
+				continue
+			} else if e.N == final && c <= e.H {
+				p++
+				return int(c-e.L) + int(e.I), p
+			}
+		}
+		i++
+	}
+	return
+}
+
+// print writes the contractTrieSet t as compilable Go code to w. It returns
+// the total number of bytes written and the size of the resulting data structure in bytes.
+func print(t *colltab.ContractTrieSet, w io.Writer, name string) (n, size int, err error) {
+	update3 := func(nn, sz int, e error) {
+		n += nn
+		if err == nil {
+			err = e
+		}
+		size += sz
+	}
+	update2 := func(nn int, e error) { update3(nn, 0, e) }
+
+	update3(printArray(*t, w, name))
+	update2(fmt.Fprintf(w, "var %sContractTrieSet = ", name))
+	update3(printStruct(*t, w, name))
+	update2(fmt.Fprintln(w))
+	return
+}
+
+func printArray(ct colltab.ContractTrieSet, w io.Writer, name string) (n, size int, err error) {
+	p := func(f string, a ...interface{}) {
+		nn, e := fmt.Fprintf(w, f, a...)
+		n += nn
+		if err == nil {
+			err = e
+		}
+	}
+	size = len(ct) * 4
+	p("// %sCTEntries: %d entries, %d bytes\n", name, len(ct), size)
+	p("var %sCTEntries = [%d]struct{L,H,N,I uint8}{\n", name, len(ct))
+	for _, fe := range ct {
+		p("\t{0x%X, 0x%X, %d, %d},\n", fe.L, fe.H, fe.N, fe.I)
+	}
+	p("}\n")
+	return
+}
+
+func printStruct(ct colltab.ContractTrieSet, w io.Writer, name string) (n, size int, err error) {
+	n, err = fmt.Fprintf(w, "colltab.ContractTrieSet( %sCTEntries[:] )", name)
+	size = int(reflect.TypeOf(ct).Size())
+	return
+}