ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/internal/colltab/trie_test.go

@@ -0,0 +1,106 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"testing"
+)
+
+// We take the smallest, largest and an arbitrary value for each
+// of the UTF-8 sequence lengths.
+var testRunes = []rune{
+	0x01, 0x0C, 0x7F, // 1-byte sequences
+	0x80, 0x100, 0x7FF, // 2-byte sequences
+	0x800, 0x999, 0xFFFF, // 3-byte sequences
+	0x10000, 0x10101, 0x10FFFF, // 4-byte sequences
+	0x200, 0x201, 0x202, 0x210, 0x215, // five entries in one sparse block
+}
+
+// Test cases for illegal runes.
+type trietest struct {
+	size  int
+	bytes []byte
+}
+
+var tests = []trietest{
+	// illegal runes
+	{1, []byte{0x80}},
+	{1, []byte{0xFF}},
+	{1, []byte{t2, tx - 1}},
+	{1, []byte{t2, t2}},
+	{2, []byte{t3, tx, tx - 1}},
+	{2, []byte{t3, tx, t2}},
+	{1, []byte{t3, tx - 1, tx}},
+	{3, []byte{t4, tx, tx, tx - 1}},
+	{3, []byte{t4, tx, tx, t2}},
+	{1, []byte{t4, t2, tx, tx - 1}},
+	{2, []byte{t4, tx, t2, tx - 1}},
+
+	// short runes
+	{0, []byte{t2}},
+	{0, []byte{t3, tx}},
+	{0, []byte{t4, tx, tx}},
+
+	// we only support UTF-8 up to utf8.UTFMax bytes (4 bytes)
+	{1, []byte{t5, tx, tx, tx, tx}},
+	{1, []byte{t6, tx, tx, tx, tx, tx}},
+}
+
+func TestLookupTrie(t *testing.T) {
+	for i, r := range testRunes {
+		b := []byte(string(r))
+		v, sz := testTrie.lookup(b)
+		if int(v) != i {
+			t.Errorf("lookup(%U): found value %#x, expected %#x", r, v, i)
+		}
+		if sz != len(b) {
+			t.Errorf("lookup(%U): found size %d, expected %d", r, sz, len(b))
+		}
+	}
+	for i, tt := range tests {
+		v, sz := testTrie.lookup(tt.bytes)
+		if int(v) != 0 {
+			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
+		}
+		if sz != tt.size {
+			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
+		}
+	}
+}
+
+// test data is taken from exp/collate/locale/build/trie_test.go
+var testValues = [832]uint32{
+	0x000c: 0x00000001,
+	0x007f: 0x00000002,
+	0x00c0: 0x00000003,
+	0x0100: 0x00000004,
+	0x0140: 0x0000000c, 0x0141: 0x0000000d, 0x0142: 0x0000000e,
+	0x0150: 0x0000000f,
+	0x0155: 0x00000010,
+	0x01bf: 0x00000005,
+	0x01c0: 0x00000006,
+	0x0219: 0x00000007,
+	0x027f: 0x00000008,
+	0x0280: 0x00000009,
+	0x02c1: 0x0000000a,
+	0x033f: 0x0000000b,
+}
+
+var testLookup = [640]uint16{
+	0x0e0: 0x05, 0x0e6: 0x06,
+	0x13f: 0x07,
+	0x140: 0x08, 0x144: 0x09,
+	0x190: 0x03,
+	0x1ff: 0x0a,
+	0x20f: 0x05,
+	0x242: 0x01, 0x244: 0x02,
+	0x248: 0x03,
+	0x25f: 0x04,
+	0x260: 0x01,
+	0x26f: 0x02,
+	0x270: 0x04, 0x274: 0x06,
+}
+
+var testTrie = Trie{testLookup[6*blockSize:], testValues[:], testLookup[:], testValues[:]}

data/ext/vendor/golang.org/x/text/internal/colltab/weighter.go

@@ -0,0 +1,31 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab // import "golang.org/x/text/internal/colltab"
+
+// A Weighter can be used as a source for Collator and Searcher.
+type Weighter interface {
+	// Start finds the start of the segment that includes position p.
+	Start(p int, b []byte) int
+
+	// StartString finds the start of the segment that includes position p.
+	StartString(p int, s string) int
+
+	// AppendNext appends Elems to buf corresponding to the longest match
+	// of a single character or contraction from the start of s.
+	// It returns the new buf and the number of bytes consumed.
+	AppendNext(buf []Elem, s []byte) (ce []Elem, n int)
+
+	// AppendNextString appends Elems to buf corresponding to the longest match
+	// of a single character or contraction from the start of s.
+	// It returns the new buf and the number of bytes consumed.
+	AppendNextString(buf []Elem, s string) (ce []Elem, n int)
+
+	// Domain returns a slice of all single characters and contractions for which
+	// collation elements are defined in this table.
+	Domain() []string
+
+	// Top returns the highest variable primary value.
+	Top() uint32
+}

data/ext/vendor/golang.org/x/text/internal/colltab/weighter_test.go

@@ -0,0 +1,42 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+// testWeighter is a simple Weighter that returns weights from a user-defined map.
+type testWeighter map[string][]Elem
+
+func (t testWeighter) Start(int, []byte) int       { return 0 }
+func (t testWeighter) StartString(int, string) int { return 0 }
+func (t testWeighter) Domain() []string            { return nil }
+func (t testWeighter) Top() uint32                 { return 0 }
+
+// maxContractBytes is the maximum length of any key in the map.
+const maxContractBytes = 10
+
+func (t testWeighter) AppendNext(buf []Elem, s []byte) ([]Elem, int) {
+	n := len(s)
+	if n > maxContractBytes {
+		n = maxContractBytes
+	}
+	for i := n; i > 0; i-- {
+		if e, ok := t[string(s[:i])]; ok {
+			return append(buf, e...), i
+		}
+	}
+	panic("incomplete testWeighter: could not find " + string(s))
+}
+
+func (t testWeighter) AppendNextString(buf []Elem, s string) ([]Elem, int) {
+	n := len(s)
+	if n > maxContractBytes {
+		n = maxContractBytes
+	}
+	for i := n; i > 0; i-- {
+		if e, ok := t[s[:i]]; ok {
+			return append(buf, e...), i
+		}
+	}
+	panic("incomplete testWeighter: could not find " + s)
+}

data/ext/vendor/golang.org/x/text/internal/export/README

@@ -0,0 +1,4 @@
+The export directory contains packages that are generated using the x/text
+infrastructure, but live elsewhere.
+At some point we can expose some of the infrastructure, but for now this
+is not done.

data/ext/vendor/golang.org/x/text/internal/export/idna/common_test.go

@@ -0,0 +1,55 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package idna
+
+// This file contains code that is common between the generation code and the
+// package's test code.
+
+import (
+	"log"
+
+	"golang.org/x/text/internal/ucd"
+)
+
+func catFromEntry(p *ucd.Parser) (cat category) {
+	r := p.Rune(0)
+	switch s := p.String(1); s {
+	case "valid":
+		cat = valid
+	case "disallowed":
+		cat = disallowed
+	case "disallowed_STD3_valid":
+		cat = disallowedSTD3Valid
+	case "disallowed_STD3_mapped":
+		cat = disallowedSTD3Mapped
+	case "mapped":
+		cat = mapped
+	case "deviation":
+		cat = deviation
+	case "ignored":
+		cat = ignored
+	default:
+		log.Fatalf("%U: Unknown category %q", r, s)
+	}
+	if s := p.String(3); s != "" {
+		if cat != valid {
+			log.Fatalf(`%U: %s defined for %q; want "valid"`, r, s, p.String(1))
+		}
+		switch s {
+		case "NV8":
+			cat = validNV8
+		case "XV8":
+			cat = validXV8
+		default:
+			log.Fatalf("%U: Unexpected exception %q", r, s)
+		}
+	}
+	return cat
+}
+
+var joinType = map[string]info{
+	"L": joiningL,
+	"D": joiningD,
+	"T": joiningT,
+	"R": joiningR,
+}

data/ext/vendor/golang.org/x/text/internal/export/idna/example_test.go

@@ -0,0 +1,68 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package idna_test
+
+import (
+	"fmt"
+
+	"golang.org/x/text/internal/export/idna"
+)
+
+func ExampleProfile() {
+	// Raw Punycode has no restrictions and does no mappings.
+	fmt.Println(idna.ToASCII(""))
+	fmt.Println(idna.ToASCII("*.faß.com"))
+	fmt.Println(idna.Punycode.ToASCII("*.faß.com"))
+
+	// Rewrite IDN for lookup. This (currently) uses transitional mappings to
+	// find a balance between IDNA2003 and IDNA2008 compatibility.
+	fmt.Println(idna.Lookup.ToASCII(""))
+	fmt.Println(idna.Lookup.ToASCII("www.faß.com"))
+
+	// Convert an IDN to ASCII for registration purposes. This changes the
+	// encoding, but reports an error if the input was illformed.
+	fmt.Println(idna.Registration.ToASCII(""))
+	fmt.Println(idna.Registration.ToASCII("www.faß.com"))
+
+	// Output:
+	//  <nil>
+	// *.xn--fa-hia.com <nil>
+	// *.xn--fa-hia.com <nil>
+	//  <nil>
+	// www.fass.com <nil>
+	//  idna: invalid label ""
+	// www.xn--fa-hia.com <nil>
+}
+
+func ExampleNew() {
+	var p *idna.Profile
+
+	// Raw Punycode has no restrictions and does no mappings.
+	p = idna.New()
+	fmt.Println(p.ToASCII("*.faß.com"))
+
+	// Do mappings. Note that star is not allowed in a DNS lookup.
+	p = idna.New(
+		idna.MapForLookup(),
+		idna.Transitional(true)) // Map ß -> ss
+	fmt.Println(p.ToASCII("*.faß.com"))
+
+	// Lookup for registration. Also does not allow '*'.
+	p = idna.New(idna.ValidateForRegistration())
+	fmt.Println(p.ToUnicode("*.faß.com"))
+
+	// Set up a profile maps for lookup, but allows wild cards.
+	p = idna.New(
+		idna.MapForLookup(),
+		idna.Transitional(true),      // Map ß -> ss
+		idna.StrictDomainName(false)) // Set more permissive ASCII rules.
+	fmt.Println(p.ToASCII("*.faß.com"))
+
+	// Output:
+	// *.xn--fa-hia.com <nil>
+	// *.fass.com idna: disallowed rune U+002A
+	// *.faß.com idna: disallowed rune U+002A
+	// *.fass.com <nil>
+}

data/ext/vendor/golang.org/x/text/internal/export/idna/gen.go

@@ -0,0 +1,276 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// This program generates the trie for idna operations. The Unicode casing
+// algorithm requires the lookup of various properties and mappings for each
+// rune. The table generated by this generator combines several of the most
+// frequently used of these into a single trie so that they can be accessed
+// with a single lookup.
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"unicode"
+	"unicode/utf8"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/triegen"
+	"golang.org/x/text/internal/ucd"
+	"golang.org/x/text/unicode/bidi"
+)
+
+func main() {
+	gen.Init()
+	genTables()
+	gen.Repackage("gen_trieval.go", "trieval.go", "idna")
+	gen.Repackage("gen_common.go", "common_test.go", "idna")
+}
+
+var runes = map[rune]info{}
+
+func genTables() {
+	t := triegen.NewTrie("idna")
+
+	ucd.Parse(gen.OpenUCDFile("DerivedNormalizationProps.txt"), func(p *ucd.Parser) {
+		r := p.Rune(0)
+		if p.String(1) == "NFC_QC" { // p.String(2) is "N" or "M"
+			runes[r] = mayNeedNorm
+		}
+	})
+	ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) {
+		r := p.Rune(0)
+
+		const cccVirama = 9
+		if p.Int(ucd.CanonicalCombiningClass) == cccVirama {
+			runes[p.Rune(0)] = viramaModifier
+		}
+		switch {
+		case unicode.In(r, unicode.Mark):
+			runes[r] |= modifier | mayNeedNorm
+		}
+		// TODO: by using UnicodeData.txt we don't mark undefined codepoints
+		// that are earmarked as RTL properly. However, an undefined cp will
+		// always fail, so there is no need to store this info.
+		switch p, _ := bidi.LookupRune(r); p.Class() {
+		case bidi.R, bidi.AL, bidi.AN:
+			if x := runes[r]; x != 0 && x != mayNeedNorm {
+				log.Fatalf("%U: rune both modifier and RTL letter/number", r)
+			}
+			runes[r] = rtl
+		}
+	})
+
+	ucd.Parse(gen.OpenUCDFile("extracted/DerivedJoiningType.txt"), func(p *ucd.Parser) {
+		switch v := p.String(1); v {
+		case "L", "D", "T", "R":
+			runes[p.Rune(0)] |= joinType[v] << joinShift
+		}
+	})
+
+	ucd.Parse(gen.OpenUnicodeFile("idna", "", "IdnaMappingTable.txt"), func(p *ucd.Parser) {
+		r := p.Rune(0)
+
+		// The mappings table explicitly defines surrogates as invalid.
+		if !utf8.ValidRune(r) {
+			return
+		}
+
+		cat := catFromEntry(p)
+		isMapped := cat == mapped || cat == disallowedSTD3Mapped || cat == deviation
+		if !isMapped {
+			// Only include additional category information for non-mapped
+			// runes. The additional information is only used after mapping and
+			// the bits would clash with mapping information.
+			// TODO: it would be possible to inline this data and avoid
+			// additional lookups. This is quite tedious, though, so let's first
+			// see if we need this.
+			cat |= category(runes[r])
+		}
+
+		s := string(p.Runes(2))
+		if s != "" && !isMapped {
+			log.Fatalf("%U: Mapping with non-mapping category %d", r, cat)
+		}
+		t.Insert(r, uint64(makeEntry(r, s))+uint64(cat))
+	})
+
+	w := gen.NewCodeWriter()
+	defer w.WriteGoFile("tables.go", "idna")
+
+	gen.WriteUnicodeVersion(w)
+
+	w.WriteVar("mappings", string(mappings))
+	w.WriteVar("xorData", string(xorData))
+
+	sz, err := t.Gen(w, triegen.Compact(&normCompacter{}))
+	if err != nil {
+		log.Fatal(err)
+	}
+	w.Size += sz
+}
+
+var (
+	// mappings contains replacement strings for mapped runes, each prefixed
+	// with a byte containing the length of the following string.
+	mappings = []byte{}
+	mapCache = map[string]int{}
+
+	// xorData is like mappings, except that it contains XOR data.
+	// We split these two tables so that we don't get an overflow.
+	xorData  = []byte{}
+	xorCache = map[string]int{}
+)
+
+// makeEntry creates a trie entry.
+func makeEntry(r rune, mapped string) info {
+	orig := string(r)
+
+	if len(orig) != len(mapped) {
+		// Store the mapped value as is in the mappings table.
+		index := len(mappings)
+		if x, ok := mapCache[mapped]; ok {
+			index = x
+		} else {
+			mapCache[mapped] = index
+			mappings = append(mappings, byte(len(mapped)))
+			mappings = append(mappings, mapped...)
+		}
+		return info(index) << indexShift
+	}
+
+	// Create per-byte XOR mask.
+	var b []byte
+	for i := 0; i < len(orig); i++ {
+		b = append(b, orig[i]^mapped[i])
+	}
+
+	// Remove leading 0 bytes, but keep at least one byte.
+	for ; len(b) > 1 && b[0] == 0; b = b[1:] {
+	}
+
+	if len(b) == 1 {
+		return xorBit | inlineXOR | info(b[0])<<indexShift
+	}
+	mapped = string(b)
+
+	// Store the mapped value as is in the mappings table.
+	index := len(xorData)
+	if x, ok := xorCache[mapped]; ok {
+		index = x
+	} else {
+		xorCache[mapped] = index
+		xorData = append(xorData, byte(len(mapped)))
+		xorData = append(xorData, mapped...)
+	}
+	return xorBit | info(index)<<indexShift
+}
+
+// The following code implements a triegen.Compacter that was originally
+// designed for normalization. The IDNA table has some similarities with the
+// norm table. Using this compacter, together with the XOR pattern approach,
+// reduces the table size by roughly 100K. It can probably be compressed further
+// by also including elements of the compacter used by cases, but for now it is
+// good enough.
+
+const maxSparseEntries = 16
+
+type normCompacter struct {
+	sparseBlocks [][]uint64
+	sparseOffset []uint16
+	sparseCount  int
+}
+
+func mostFrequentStride(a []uint64) int {
+	counts := make(map[int]int)
+	var v int
+	for _, x := range a {
+		if stride := int(x) - v; v != 0 && stride >= 0 {
+			counts[stride]++
+		}
+		v = int(x)
+	}
+	var maxs, maxc int
+	for stride, cnt := range counts {
+		if cnt > maxc || (cnt == maxc && stride < maxs) {
+			maxs, maxc = stride, cnt
+		}
+	}
+	return maxs
+}
+
+func countSparseEntries(a []uint64) int {
+	stride := mostFrequentStride(a)
+	var v, count int
+	for _, tv := range a {
+		if int(tv)-v != stride {
+			if tv != 0 {
+				count++
+			}
+		}
+		v = int(tv)
+	}
+	return count
+}
+
+func (c *normCompacter) Size(v []uint64) (sz int, ok bool) {
+	if n := countSparseEntries(v); n <= maxSparseEntries {
+		return (n+1)*4 + 2, true
+	}
+	return 0, false
+}
+
+func (c *normCompacter) Store(v []uint64) uint32 {
+	h := uint32(len(c.sparseOffset))
+	c.sparseBlocks = append(c.sparseBlocks, v)
+	c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount))
+	c.sparseCount += countSparseEntries(v) + 1
+	return h
+}
+
+func (c *normCompacter) Handler() string {
+	return "idnaSparse.lookup"
+}
+
+func (c *normCompacter) Print(w io.Writer) (retErr error) {
+	p := func(f string, x ...interface{}) {
+		if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil {
+			retErr = err
+		}
+	}
+
+	ls := len(c.sparseBlocks)
+	p("// idnaSparseOffset: %d entries, %d bytes\n", ls, ls*2)
+	p("var idnaSparseOffset = %#v\n\n", c.sparseOffset)
+
+	ns := c.sparseCount
+	p("// idnaSparseValues: %d entries, %d bytes\n", ns, ns*4)
+	p("var idnaSparseValues = [%d]valueRange {", ns)
+	for i, b := range c.sparseBlocks {
+		p("\n// Block %#x, offset %#x", i, c.sparseOffset[i])
+		var v int
+		stride := mostFrequentStride(b)
+		n := countSparseEntries(b)
+		p("\n{value:%#04x,lo:%#02x},", stride, uint8(n))
+		for i, nv := range b {
+			if int(nv)-v != stride {
+				if v != 0 {
+					p(",hi:%#02x},", 0x80+i-1)
+				}
+				if nv != 0 {
+					p("\n{value:%#04x,lo:%#02x", nv, 0x80+i)
+				}
+			}
+			v = int(nv)
+		}
+		if v != 0 {
+			p(",hi:%#02x},", 0x80+len(b)-1)
+		}
+	}
+	p("\n}\n\n")
+	return
+}