ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/collate/build/contract_test.go

@@ -0,0 +1,266 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	"bytes"
+	"sort"
+	"testing"
+
+	"golang.org/x/text/internal/colltab"
+)
+
+var largetosmall = []stridx{
+	{"a", 5},
+	{"ab", 4},
+	{"abc", 3},
+	{"abcd", 2},
+	{"abcde", 1},
+	{"abcdef", 0},
+}
+
+var offsetSortTests = [][]stridx{
+	{
+		{"bcde", 1},
+		{"bc", 5},
+		{"ab", 4},
+		{"bcd", 3},
+		{"abcd", 0},
+		{"abc", 2},
+	},
+	largetosmall,
+}
+
+func TestOffsetSort(t *testing.T) {
+	for i, st := range offsetSortTests {
+		sort.Sort(offsetSort(st))
+		for j, si := range st {
+			if j != si.index {
+				t.Errorf("%d: failed: %v", i, st)
+			}
+		}
+	}
+	for i, tt := range genStateTests {
+		// ensure input is well-formed
+		sort.Sort(offsetSort(tt.in))
+		for j, si := range tt.in {
+			if si.index != j+1 {
+				t.Errorf("%dth sort failed: %v", i, tt.in)
+			}
+		}
+	}
+}
+
+var genidxtest1 = []stridx{
+	{"bcde", 3},
+	{"bc", 6},
+	{"ab", 2},
+	{"bcd", 5},
+	{"abcd", 0},
+	{"abc", 1},
+	{"bcdf", 4},
+}
+
+var genidxSortTests = [][]stridx{
+	genidxtest1,
+	largetosmall,
+}
+
+func TestGenIdxSort(t *testing.T) {
+	for i, st := range genidxSortTests {
+		sort.Sort(genidxSort(st))
+		for j, si := range st {
+			if j != si.index {
+				t.Errorf("%dth sort failed %v", i, st)
+				break
+			}
+		}
+	}
+}
+
+var entrySortTests = []colltab.ContractTrieSet{
+	{
+		{10, 0, 1, 3},
+		{99, 0, 1, 0},
+		{20, 50, 0, 2},
+		{30, 0, 1, 1},
+	},
+}
+
+func TestEntrySort(t *testing.T) {
+	for i, et := range entrySortTests {
+		sort.Sort(entrySort(et))
+		for j, fe := range et {
+			if j != int(fe.I) {
+				t.Errorf("%dth sort failed %v", i, et)
+				break
+			}
+		}
+	}
+}
+
+type GenStateTest struct {
+	in            []stridx
+	firstBlockLen int
+	out           colltab.ContractTrieSet
+}
+
+var genStateTests = []GenStateTest{
+	{[]stridx{
+		{"abc", 1},
+	},
+		1,
+		colltab.ContractTrieSet{
+			{'a', 0, 1, noIndex},
+			{'b', 0, 1, noIndex},
+			{'c', 'c', final, 1},
+		},
+	},
+	{[]stridx{
+		{"abc", 1},
+		{"abd", 2},
+		{"abe", 3},
+	},
+		1,
+		colltab.ContractTrieSet{
+			{'a', 0, 1, noIndex},
+			{'b', 0, 1, noIndex},
+			{'c', 'e', final, 1},
+		},
+	},
+	{[]stridx{
+		{"abc", 1},
+		{"ab", 2},
+		{"a", 3},
+	},
+		1,
+		colltab.ContractTrieSet{
+			{'a', 0, 1, 3},
+			{'b', 0, 1, 2},
+			{'c', 'c', final, 1},
+		},
+	},
+	{[]stridx{
+		{"abc", 1},
+		{"abd", 2},
+		{"ab", 3},
+		{"ac", 4},
+		{"a", 5},
+		{"b", 6},
+	},
+		2,
+		colltab.ContractTrieSet{
+			{'b', 'b', final, 6},
+			{'a', 0, 2, 5},
+			{'c', 'c', final, 4},
+			{'b', 0, 1, 3},
+			{'c', 'd', final, 1},
+		},
+	},
+	{[]stridx{
+		{"bcde", 2},
+		{"bc", 7},
+		{"ab", 6},
+		{"bcd", 5},
+		{"abcd", 1},
+		{"abc", 4},
+		{"bcdf", 3},
+	},
+		2,
+		colltab.ContractTrieSet{
+			{'b', 3, 1, noIndex},
+			{'a', 0, 1, noIndex},
+			{'b', 0, 1, 6},
+			{'c', 0, 1, 4},
+			{'d', 'd', final, 1},
+			{'c', 0, 1, 7},
+			{'d', 0, 1, 5},
+			{'e', 'f', final, 2},
+		},
+	},
+}
+
+func TestGenStates(t *testing.T) {
+	for i, tt := range genStateTests {
+		si := []stridx{}
+		for _, e := range tt.in {
+			si = append(si, e)
+		}
+		// ensure input is well-formed
+		sort.Sort(genidxSort(si))
+		ct := colltab.ContractTrieSet{}
+		n, _ := genStates(&ct, si)
+		if nn := tt.firstBlockLen; nn != n {
+			t.Errorf("%d: block len %v; want %v", i, n, nn)
+		}
+		if lv, lw := len(ct), len(tt.out); lv != lw {
+			t.Errorf("%d: len %v; want %v", i, lv, lw)
+			continue
+		}
+		for j, fe := range tt.out {
+			const msg = "%d:%d: value %s=%v; want %v"
+			if fe.L != ct[j].L {
+				t.Errorf(msg, i, j, "l", ct[j].L, fe.L)
+			}
+			if fe.H != ct[j].H {
+				t.Errorf(msg, i, j, "h", ct[j].H, fe.H)
+			}
+			if fe.N != ct[j].N {
+				t.Errorf(msg, i, j, "n", ct[j].N, fe.N)
+			}
+			if fe.I != ct[j].I {
+				t.Errorf(msg, i, j, "i", ct[j].I, fe.I)
+			}
+		}
+	}
+}
+
+func TestLookupContraction(t *testing.T) {
+	for i, tt := range genStateTests {
+		input := []string{}
+		for _, e := range tt.in {
+			input = append(input, e.str)
+		}
+		cts := colltab.ContractTrieSet{}
+		h, _ := appendTrie(&cts, input)
+		for j, si := range tt.in {
+			str := si.str
+			for _, s := range []string{str, str + "X"} {
+				msg := "%d:%d: %s(%s) %v; want %v"
+				idx, sn := lookup(&cts, h, []byte(s))
+				if idx != si.index {
+					t.Errorf(msg, i, j, "index", s, idx, si.index)
+				}
+				if sn != len(str) {
+					t.Errorf(msg, i, j, "sn", s, sn, len(str))
+				}
+			}
+		}
+	}
+}
+
+func TestPrintContractionTrieSet(t *testing.T) {
+	testdata := colltab.ContractTrieSet(genStateTests[4].out)
+	buf := &bytes.Buffer{}
+	print(&testdata, buf, "test")
+	if contractTrieOutput != buf.String() {
+		t.Errorf("output differs; found\n%s", buf.String())
+		println(string(buf.Bytes()))
+	}
+}
+
+const contractTrieOutput = `// testCTEntries: 8 entries, 32 bytes
+var testCTEntries = [8]struct{L,H,N,I uint8}{
+	{0x62, 0x3, 1, 255},
+	{0x61, 0x0, 1, 255},
+	{0x62, 0x0, 1, 6},
+	{0x63, 0x0, 1, 4},
+	{0x64, 0x64, 0, 1},
+	{0x63, 0x0, 1, 7},
+	{0x64, 0x0, 1, 5},
+	{0x65, 0x66, 0, 2},
+}
+var testContractTrieSet = colltab.ContractTrieSet( testCTEntries[:] )
+`

data/ext/vendor/golang.org/x/text/collate/build/order.go

@@ -0,0 +1,393 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/unicode/norm"
+)
+
+type logicalAnchor int
+
+const (
+	firstAnchor logicalAnchor = -1
+	noAnchor                  = 0
+	lastAnchor                = 1
+)
+
+// entry is used to keep track of a single entry in the collation element table
+// during building. Examples of entries can be found in the Default Unicode
+// Collation Element Table.
+// See http://www.unicode.org/Public/UCA/6.0.0/allkeys.txt.
+type entry struct {
+	str    string // same as string(runes)
+	runes  []rune
+	elems  []rawCE // the collation elements
+	extend string  // weights of extend to be appended to elems
+	before bool    // weights relative to next instead of previous.
+	lock   bool    // entry is used in extension and can no longer be moved.
+
+	// prev, next, and level are used to keep track of tailorings.
+	prev, next *entry
+	level      colltab.Level // next differs at this level
+	skipRemove bool          // do not unlink when removed
+
+	decompose bool // can use NFKD decomposition to generate elems
+	exclude   bool // do not include in table
+	implicit  bool // derived, is not included in the list
+	modified  bool // entry was modified in tailoring
+	logical   logicalAnchor
+
+	expansionIndex    int // used to store index into expansion table
+	contractionHandle ctHandle
+	contractionIndex  int // index into contraction elements
+}
+
+func (e *entry) String() string {
+	return fmt.Sprintf("%X (%q) -> %X (ch:%x; ci:%d, ei:%d)",
+		e.runes, e.str, e.elems, e.contractionHandle, e.contractionIndex, e.expansionIndex)
+}
+
+func (e *entry) skip() bool {
+	return e.contraction()
+}
+
+func (e *entry) expansion() bool {
+	return !e.decompose && len(e.elems) > 1
+}
+
+func (e *entry) contraction() bool {
+	return len(e.runes) > 1
+}
+
+func (e *entry) contractionStarter() bool {
+	return e.contractionHandle.n != 0
+}
+
+// nextIndexed gets the next entry that needs to be stored in the table.
+// It returns the entry and the collation level at which the next entry differs
+// from the current entry.
+// Entries that can be explicitly derived and logical reset positions are
+// examples of entries that will not be indexed.
+func (e *entry) nextIndexed() (*entry, colltab.Level) {
+	level := e.level
+	for e = e.next; e != nil && (e.exclude || len(e.elems) == 0); e = e.next {
+		if e.level < level {
+			level = e.level
+		}
+	}
+	return e, level
+}
+
+// remove unlinks entry e from the sorted chain and clears the collation
+// elements. e may not be at the front or end of the list. This should always
+// be the case, as the front and end of the list are always logical anchors,
+// which may not be removed.
+func (e *entry) remove() {
+	if e.logical != noAnchor {
+		log.Fatalf("may not remove anchor %q", e.str)
+	}
+	// TODO: need to set e.prev.level to e.level if e.level is smaller?
+	e.elems = nil
+	if !e.skipRemove {
+		if e.prev != nil {
+			e.prev.next = e.next
+		}
+		if e.next != nil {
+			e.next.prev = e.prev
+		}
+	}
+	e.skipRemove = false
+}
+
+// insertAfter inserts n after e.
+func (e *entry) insertAfter(n *entry) {
+	if e == n {
+		panic("e == anchor")
+	}
+	if e == nil {
+		panic("unexpected nil anchor")
+	}
+	n.remove()
+	n.decompose = false // redo decomposition test
+
+	n.next = e.next
+	n.prev = e
+	if e.next != nil {
+		e.next.prev = n
+	}
+	e.next = n
+}
+
+// insertBefore inserts n before e.
+func (e *entry) insertBefore(n *entry) {
+	if e == n {
+		panic("e == anchor")
+	}
+	if e == nil {
+		panic("unexpected nil anchor")
+	}
+	n.remove()
+	n.decompose = false // redo decomposition test
+
+	n.prev = e.prev
+	n.next = e
+	if e.prev != nil {
+		e.prev.next = n
+	}
+	e.prev = n
+}
+
+func (e *entry) encodeBase() (ce uint32, err error) {
+	switch {
+	case e.expansion():
+		ce, err = makeExpandIndex(e.expansionIndex)
+	default:
+		if e.decompose {
+			log.Fatal("decompose should be handled elsewhere")
+		}
+		ce, err = makeCE(e.elems[0])
+	}
+	return
+}
+
+func (e *entry) encode() (ce uint32, err error) {
+	if e.skip() {
+		log.Fatal("cannot build colElem for entry that should be skipped")
+	}
+	switch {
+	case e.decompose:
+		t1 := e.elems[0].w[2]
+		t2 := 0
+		if len(e.elems) > 1 {
+			t2 = e.elems[1].w[2]
+		}
+		ce, err = makeDecompose(t1, t2)
+	case e.contractionStarter():
+		ce, err = makeContractIndex(e.contractionHandle, e.contractionIndex)
+	default:
+		if len(e.runes) > 1 {
+			log.Fatal("colElem: contractions are handled in contraction trie")
+		}
+		ce, err = e.encodeBase()
+	}
+	return
+}
+
+// entryLess returns true if a sorts before b and false otherwise.
+func entryLess(a, b *entry) bool {
+	if res, _ := compareWeights(a.elems, b.elems); res != 0 {
+		return res == -1
+	}
+	if a.logical != noAnchor {
+		return a.logical == firstAnchor
+	}
+	if b.logical != noAnchor {
+		return b.logical == lastAnchor
+	}
+	return a.str < b.str
+}
+
+type sortedEntries []*entry
+
+func (s sortedEntries) Len() int {
+	return len(s)
+}
+
+func (s sortedEntries) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s sortedEntries) Less(i, j int) bool {
+	return entryLess(s[i], s[j])
+}
+
+type ordering struct {
+	id       string
+	entryMap map[string]*entry
+	ordered  []*entry
+	handle   *trieHandle
+}
+
+// insert inserts e into both entryMap and ordered.
+// Note that insert simply appends e to ordered.  To reattain a sorted
+// order, o.sort() should be called.
+func (o *ordering) insert(e *entry) {
+	if e.logical == noAnchor {
+		o.entryMap[e.str] = e
+	} else {
+		// Use key format as used in UCA rules.
+		o.entryMap[fmt.Sprintf("[%s]", e.str)] = e
+		// Also add index entry for XML format.
+		o.entryMap[fmt.Sprintf("<%s/>", strings.Replace(e.str, " ", "_", -1))] = e
+	}
+	o.ordered = append(o.ordered, e)
+}
+
+// newEntry creates a new entry for the given info and inserts it into
+// the index.
+func (o *ordering) newEntry(s string, ces []rawCE) *entry {
+	e := &entry{
+		runes: []rune(s),
+		elems: ces,
+		str:   s,
+	}
+	o.insert(e)
+	return e
+}
+
+// find looks up and returns the entry for the given string.
+// It returns nil if str is not in the index and if an implicit value
+// cannot be derived, that is, if str represents more than one rune.
+func (o *ordering) find(str string) *entry {
+	e := o.entryMap[str]
+	if e == nil {
+		r := []rune(str)
+		if len(r) == 1 {
+			const (
+				firstHangul = 0xAC00
+				lastHangul  = 0xD7A3
+			)
+			if r[0] >= firstHangul && r[0] <= lastHangul {
+				ce := []rawCE{}
+				nfd := norm.NFD.String(str)
+				for _, r := range nfd {
+					ce = append(ce, o.find(string(r)).elems...)
+				}
+				e = o.newEntry(nfd, ce)
+			} else {
+				e = o.newEntry(string(r[0]), []rawCE{
+					{w: []int{
+						implicitPrimary(r[0]),
+						defaultSecondary,
+						defaultTertiary,
+						int(r[0]),
+					},
+					},
+				})
+				e.modified = true
+			}
+			e.exclude = true // do not index implicits
+		}
+	}
+	return e
+}
+
+// makeRootOrdering returns a newly initialized ordering value and populates
+// it with a set of logical reset points that can be used as anchors.
+// The anchors first_tertiary_ignorable and __END__ will always sort at
+// the beginning and end, respectively. This means that prev and next are non-nil
+// for any indexed entry.
+func makeRootOrdering() ordering {
+	const max = unicode.MaxRune
+	o := ordering{
+		entryMap: make(map[string]*entry),
+	}
+	insert := func(typ logicalAnchor, s string, ce []int) {
+		e := &entry{
+			elems:   []rawCE{{w: ce}},
+			str:     s,
+			exclude: true,
+			logical: typ,
+		}
+		o.insert(e)
+	}
+	insert(firstAnchor, "first tertiary ignorable", []int{0, 0, 0, 0})
+	insert(lastAnchor, "last tertiary ignorable", []int{0, 0, 0, max})
+	insert(lastAnchor, "last primary ignorable", []int{0, defaultSecondary, defaultTertiary, max})
+	insert(lastAnchor, "last non ignorable", []int{maxPrimary, defaultSecondary, defaultTertiary, max})
+	insert(lastAnchor, "__END__", []int{1 << maxPrimaryBits, defaultSecondary, defaultTertiary, max})
+	return o
+}
+
+// patchForInsert eleminates entries from the list with more than one collation element.
+// The next and prev fields of the eliminated entries still point to appropriate
+// values in the newly created list.
+// It requires that sort has been called.
+func (o *ordering) patchForInsert() {
+	for i := 0; i < len(o.ordered)-1; {
+		e := o.ordered[i]
+		lev := e.level
+		n := e.next
+		for ; n != nil && len(n.elems) > 1; n = n.next {
+			if n.level < lev {
+				lev = n.level
+			}
+			n.skipRemove = true
+		}
+		for ; o.ordered[i] != n; i++ {
+			o.ordered[i].level = lev
+			o.ordered[i].next = n
+			o.ordered[i+1].prev = e
+		}
+	}
+}
+
+// clone copies all ordering of es into a new ordering value.
+func (o *ordering) clone() *ordering {
+	o.sort()
+	oo := ordering{
+		entryMap: make(map[string]*entry),
+	}
+	for _, e := range o.ordered {
+		ne := &entry{
+			runes:     e.runes,
+			elems:     e.elems,
+			str:       e.str,
+			decompose: e.decompose,
+			exclude:   e.exclude,
+			logical:   e.logical,
+		}
+		oo.insert(ne)
+	}
+	oo.sort() // link all ordering.
+	oo.patchForInsert()
+	return &oo
+}
+
+// front returns the first entry to be indexed.
+// It assumes that sort() has been called.
+func (o *ordering) front() *entry {
+	e := o.ordered[0]
+	if e.prev != nil {
+		log.Panicf("unexpected first entry: %v", e)
+	}
+	// The first entry is always a logical position, which should not be indexed.
+	e, _ = e.nextIndexed()
+	return e
+}
+
+// sort sorts all ordering based on their collation elements and initializes
+// the prev, next, and level fields accordingly.
+func (o *ordering) sort() {
+	sort.Sort(sortedEntries(o.ordered))
+	l := o.ordered
+	for i := 1; i < len(l); i++ {
+		k := i - 1
+		l[k].next = l[i]
+		_, l[k].level = compareWeights(l[k].elems, l[i].elems)
+		l[i].prev = l[k]
+	}
+}
+
+// genColElems generates a collation element array from the runes in str. This
+// assumes that all collation elements have already been added to the Builder.
+func (o *ordering) genColElems(str string) []rawCE {
+	elems := []rawCE{}
+	for _, r := range []rune(str) {
+		for _, ce := range o.find(string(r)).elems {
+			if ce.w[0] != 0 || ce.w[1] != 0 || ce.w[2] != 0 {
+				elems = append(elems, ce)
+			}
+		}
+	}
+	return elems
+}