ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/internal/triegen/example_compact_test.go

@@ -0,0 +1,71 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package triegen_test
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"golang.org/x/text/internal/triegen"
+)
+
+func ExampleCompacter() {
+	t := triegen.NewTrie("root")
+	for r := rune(0); r < 10000; r += 64 {
+		t.Insert(r, 0x9015BADA55^uint64(r))
+	}
+	sz, _ := t.Gen(ioutil.Discard)
+
+	fmt.Printf("Size normal:    %5d\n", sz)
+
+	var c myCompacter
+	sz, _ = t.Gen(ioutil.Discard, triegen.Compact(&c))
+
+	fmt.Printf("Size compacted: %5d\n", sz)
+
+	// Output:
+	// Size normal:    81344
+	// Size compacted:  3224
+}
+
+// A myCompacter accepts a block if only the first value is given.
+type myCompacter []uint64
+
+func (c *myCompacter) Size(values []uint64) (sz int, ok bool) {
+	for _, v := range values[1:] {
+		if v != 0 {
+			return 0, false
+		}
+	}
+	return 8, true // the size of a uint64
+}
+
+func (c *myCompacter) Store(v []uint64) uint32 {
+	x := uint32(len(*c))
+	*c = append(*c, v[0])
+	return x
+}
+
+func (c *myCompacter) Print(w io.Writer) error {
+	fmt.Fprintln(w, "var firstValue = []uint64{")
+	for _, v := range *c {
+		fmt.Fprintf(w, "\t%#x,\n", v)
+	}
+	fmt.Fprintln(w, "}")
+	return nil
+}
+
+func (c *myCompacter) Handler() string {
+	return "getFirstValue"
+
+	// Where getFirstValue is included along with the generated code:
+	// func getFirstValue(n uint32, b byte) uint64 {
+	//     if b == 0x80 { // the first continuation byte
+	//         return firstValue[n]
+	//     }
+	//     return 0
+	//  }
+}

data/ext/vendor/golang.org/x/text/internal/triegen/example_test.go

@@ -0,0 +1,148 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package triegen_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"unicode"
+
+	"golang.org/x/text/internal/triegen"
+)
+
+const seed = 0x12345
+
+var genWriter = ioutil.Discard
+
+func randomRunes() map[rune]uint8 {
+	rnd := rand.New(rand.NewSource(seed))
+	m := map[rune]uint8{}
+	for len(m) < 100 {
+		// Only set our random rune if it is a valid Unicode code point.
+		if r := rune(rnd.Int31n(unicode.MaxRune + 1)); []rune(string(r))[0] == r {
+			m[r] = 1
+		}
+	}
+	return m
+}
+
+// Example_build shows how to build a simple trie. It assigns the value 1 to
+// 100 random runes generated by randomRunes.
+func Example_build() {
+	t := triegen.NewTrie("rand")
+
+	for r, _ := range randomRunes() {
+		t.Insert(r, 1)
+	}
+	sz, err := t.Gen(genWriter)
+
+	fmt.Printf("Trie size: %d bytes\n", sz)
+	fmt.Printf("Error:     %v\n", err)
+
+	// Output:
+	// Trie size: 9280 bytes
+	// Error:     <nil>
+}
+
+// Example_lookup demonstrates how to use the trie generated by Example_build.
+func Example_lookup() {
+	trie := newRandTrie(0)
+
+	// The same set of runes used by Example_build.
+	runes := randomRunes()
+
+	// Verify the right value is returned for all runes.
+	for r := rune(0); r <= unicode.MaxRune; r++ {
+		// Note that the return type of lookup is uint8.
+		if v, _ := trie.lookupString(string(r)); v != runes[r] {
+			fmt.Println("FAILURE")
+			return
+		}
+	}
+	fmt.Println("SUCCESS")
+
+	// Output:
+	// SUCCESS
+}
+
+// runeValues generates some random values for a set of interesting runes.
+func runeValues() map[rune]uint64 {
+	rnd := rand.New(rand.NewSource(seed))
+	m := map[rune]uint64{}
+	for p := 4; p <= unicode.MaxRune; p <<= 1 {
+		for d := -1; d <= 1; d++ {
+			m[rune(p+d)] = uint64(rnd.Int63())
+		}
+	}
+	return m
+}
+
+// ExampleGen_build demonstrates the creation of multiple tries sharing common
+// blocks. ExampleGen_lookup demonstrates how to use the generated tries.
+func ExampleGen_build() {
+	var tries []*triegen.Trie
+
+	rv := runeValues()
+	for _, c := range []struct {
+		include func(rune) bool
+		name    string
+	}{
+		{func(r rune) bool { return true }, "all"},
+		{func(r rune) bool { return r < 0x80 }, "ASCII only"},
+		{func(r rune) bool { return r < 0x80 }, "ASCII only 2"},
+		{func(r rune) bool { return r <= 0xFFFF }, "BMP only"},
+		{func(r rune) bool { return r > 0xFFFF }, "No BMP"},
+	} {
+		t := triegen.NewTrie(c.name)
+		tries = append(tries, t)
+
+		for r, v := range rv {
+			if c.include(r) {
+				t.Insert(r, v)
+			}
+		}
+	}
+	sz, err := triegen.Gen(genWriter, "multi", tries)
+
+	fmt.Printf("Trie size: %d bytes\n", sz)
+	fmt.Printf("Error:     %v\n", err)
+
+	// Output:
+	// Trie size: 18250 bytes
+	// Error:     <nil>
+}
+
+// ExampleGen_lookup shows how to look up values in the trie generated by
+// ExampleGen_build.
+func ExampleGen_lookup() {
+	rv := runeValues()
+	for i, include := range []func(rune) bool{
+		func(r rune) bool { return true },        // all
+		func(r rune) bool { return r < 0x80 },    // ASCII only
+		func(r rune) bool { return r < 0x80 },    // ASCII only 2
+		func(r rune) bool { return r <= 0xFFFF }, // BMP only
+		func(r rune) bool { return r > 0xFFFF },  // No BMP
+	} {
+		t := newMultiTrie(i)
+
+		for r := rune(0); r <= unicode.MaxRune; r++ {
+			x := uint64(0)
+			if include(r) {
+				x = rv[r]
+			}
+			// As we convert from a valid rune, we know it is safe to use
+			// lookupStringUnsafe.
+			if v := t.lookupStringUnsafe(string(r)); x != v {
+				fmt.Println("FAILURE")
+				return
+			}
+		}
+	}
+	fmt.Println("SUCCESS")
+
+	// Output:
+	// SUCCESS
+}

data/ext/vendor/golang.org/x/text/internal/triegen/gen_test.go

@@ -0,0 +1,68 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build generate
+
+package triegen_test
+
+// The code in this file generates captures and writes the tries generated in
+// the examples to data_test.go. To invoke it, run:
+// 		go test -tags=generate
+//
+// Making the generation code a "test" allows us to link in the necessary test
+// code.
+
+import (
+	"log"
+	"os"
+	"os/exec"
+)
+
+func init() {
+	const tmpfile = "tmpout"
+	const dstfile = "data_test.go"
+
+	f, err := os.Create(tmpfile)
+	if err != nil {
+		log.Fatalf("Could not create output file: %v", err)
+	}
+	defer os.Remove(tmpfile)
+	defer f.Close()
+
+	// We exit before this function returns, regardless of success or failure,
+	// so there's no need to save (and later restore) the existing genWriter
+	// value.
+	genWriter = f
+
+	f.Write([]byte(header))
+
+	Example_build()
+	ExampleGen_build()
+
+	if err := exec.Command("gofmt", "-w", tmpfile).Run(); err != nil {
+		log.Fatal(err)
+	}
+	os.Remove(dstfile)
+	os.Rename(tmpfile, dstfile)
+
+	os.Exit(0)
+}
+
+const header = `// This file is generated with "go test -tags generate". DO NOT EDIT!
+// +build !generate
+
+package triegen_test
+`
+
+// Stubs for generated tries. These are needed as we exclude data_test.go if
+// the generate flag is set. This will clearly make the tests fail, but that
+// is okay. It allows us to bootstrap.
+
+type trie struct{}
+
+func (t *trie) lookupString(string) (uint8, int) { return 0, 1 }
+func (t *trie) lookupStringUnsafe(string) uint64 { return 0 }
+
+func newRandTrie(i int) *trie  { return &trie{} }
+func newMultiTrie(i int) *trie { return &trie{} }

data/ext/vendor/golang.org/x/text/internal/triegen/print.go

@@ -0,0 +1,251 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package triegen
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"text/template"
+)
+
+// print writes all the data structures as well as the code necessary to use the
+// trie to w.
+func (b *builder) print(w io.Writer) error {
+	b.Stats.NValueEntries = len(b.ValueBlocks) * blockSize
+	b.Stats.NValueBytes = len(b.ValueBlocks) * blockSize * b.ValueSize
+	b.Stats.NIndexEntries = len(b.IndexBlocks) * blockSize
+	b.Stats.NIndexBytes = len(b.IndexBlocks) * blockSize * b.IndexSize
+	b.Stats.NHandleBytes = len(b.Trie) * 2 * b.IndexSize
+
+	// If we only have one root trie, all starter blocks are at position 0 and
+	// we can access the arrays directly.
+	if len(b.Trie) == 1 {
+		// At this point we cannot refer to the generated tables directly.
+		b.ASCIIBlock = b.Name + "Values"
+		b.StarterBlock = b.Name + "Index"
+	} else {
+		// Otherwise we need to have explicit starter indexes in the trie
+		// structure.
+		b.ASCIIBlock = "t.ascii"
+		b.StarterBlock = "t.utf8Start"
+	}
+
+	b.SourceType = "[]byte"
+	if err := lookupGen.Execute(w, b); err != nil {
+		return err
+	}
+
+	b.SourceType = "string"
+	if err := lookupGen.Execute(w, b); err != nil {
+		return err
+	}
+
+	if err := trieGen.Execute(w, b); err != nil {
+		return err
+	}
+
+	for _, c := range b.Compactions {
+		if err := c.c.Print(w); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func printValues(n int, values []uint64) string {
+	w := &bytes.Buffer{}
+	boff := n * blockSize
+	fmt.Fprintf(w, "\t// Block %#x, offset %#x", n, boff)
+	var newline bool
+	for i, v := range values {
+		if i%6 == 0 {
+			newline = true
+		}
+		if v != 0 {
+			if newline {
+				fmt.Fprintf(w, "\n")
+				newline = false
+			}
+			fmt.Fprintf(w, "\t%#02x:%#04x, ", boff+i, v)
+		}
+	}
+	return w.String()
+}
+
+func printIndex(b *builder, nr int, n *node) string {
+	w := &bytes.Buffer{}
+	boff := nr * blockSize
+	fmt.Fprintf(w, "\t// Block %#x, offset %#x", nr, boff)
+	var newline bool
+	for i, c := range n.children {
+		if i%8 == 0 {
+			newline = true
+		}
+		if c != nil {
+			v := b.Compactions[c.index.compaction].Offset + uint32(c.index.index)
+			if v != 0 {
+				if newline {
+					fmt.Fprintf(w, "\n")
+					newline = false
+				}
+				fmt.Fprintf(w, "\t%#02x:%#02x, ", boff+i, v)
+			}
+		}
+	}
+	return w.String()
+}
+
+var (
+	trieGen = template.Must(template.New("trie").Funcs(template.FuncMap{
+		"printValues": printValues,
+		"printIndex":  printIndex,
+		"title":       strings.Title,
+		"dec":         func(x int) int { return x - 1 },
+		"psize": func(n int) string {
+			return fmt.Sprintf("%d bytes (%.2f KiB)", n, float64(n)/1024)
+		},
+	}).Parse(trieTemplate))
+	lookupGen = template.Must(template.New("lookup").Parse(lookupTemplate))
+)
+
+// TODO: consider the return type of lookup. It could be uint64, even if the
+// internal value type is smaller. We will have to verify this with the
+// performance of unicode/norm, which is very sensitive to such changes.
+const trieTemplate = `{{$b := .}}{{$multi := gt (len .Trie) 1}}
+// {{.Name}}Trie. Total size: {{psize .Size}}. Checksum: {{printf "%08x" .Checksum}}.
+type {{.Name}}Trie struct { {{if $multi}}
+	ascii []{{.ValueType}} // index for ASCII bytes
+	utf8Start  []{{.IndexType}} // index for UTF-8 bytes >= 0xC0
+{{end}}}
+
+func new{{title .Name}}Trie(i int) *{{.Name}}Trie { {{if $multi}}
+	h := {{.Name}}TrieHandles[i]
+	return &{{.Name}}Trie{ {{.Name}}Values[uint32(h.ascii)<<6:], {{.Name}}Index[uint32(h.multi)<<6:] }
+}
+
+type {{.Name}}TrieHandle struct {
+	ascii, multi {{.IndexType}}
+}
+
+// {{.Name}}TrieHandles: {{len .Trie}} handles, {{.Stats.NHandleBytes}} bytes
+var {{.Name}}TrieHandles = [{{len .Trie}}]{{.Name}}TrieHandle{
+{{range .Trie}}	{ {{.ASCIIIndex}}, {{.StarterIndex}} }, // {{printf "%08x" .Checksum}}: {{.Name}}
+{{end}}}{{else}}
+	return &{{.Name}}Trie{}
+}
+{{end}}
+// lookupValue determines the type of block n and looks up the value for b.
+func (t *{{.Name}}Trie) lookupValue(n uint32, b byte) {{.ValueType}}{{$last := dec (len .Compactions)}} {
+	switch { {{range $i, $c := .Compactions}}
+		{{if eq $i $last}}default{{else}}case n < {{$c.Cutoff}}{{end}}:{{if ne $i 0}}
+			n -= {{$c.Offset}}{{end}}
+			return {{print $b.ValueType}}({{$c.Handler}}){{end}}
+	}
+}
+
+// {{.Name}}Values: {{len .ValueBlocks}} blocks, {{.Stats.NValueEntries}} entries, {{.Stats.NValueBytes}} bytes
+// The third block is the zero block.
+var {{.Name}}Values = [{{.Stats.NValueEntries}}]{{.ValueType}} {
+{{range $i, $v := .ValueBlocks}}{{printValues $i $v}}
+{{end}}}
+
+// {{.Name}}Index: {{len .IndexBlocks}} blocks, {{.Stats.NIndexEntries}} entries, {{.Stats.NIndexBytes}} bytes
+// Block 0 is the zero block.
+var {{.Name}}Index = [{{.Stats.NIndexEntries}}]{{.IndexType}} {
+{{range $i, $v := .IndexBlocks}}{{printIndex $b $i $v}}
+{{end}}}
+`
+
+// TODO: consider allowing zero-length strings after evaluating performance with
+// unicode/norm.
+const lookupTemplate = `
+// lookup{{if eq .SourceType "string"}}String{{end}} returns the trie value for the first UTF-8 encoding in s and
+// the width in bytes of this encoding. The size will be 0 if s does not
+// hold enough bytes to complete the encoding. len(s) must be greater than 0.
+func (t *{{.Name}}Trie) lookup{{if eq .SourceType "string"}}String{{end}}(s {{.SourceType}}) (v {{.ValueType}}, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return {{.ASCIIBlock}}[c0], 1
+	case c0 < 0xC2:
+		return 0, 1  // Illegal UTF-8: not a starter, not ASCII.
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return 0, 0
+		}
+		i := {{.StarterBlock}}[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return 0, 1 // Illegal UTF-8: not a continuation byte.
+		}
+		return t.lookupValue(uint32(i), c1), 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return 0, 0
+		}
+		i := {{.StarterBlock}}[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return 0, 1 // Illegal UTF-8: not a continuation byte.
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = {{.Name}}Index[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return 0, 2 // Illegal UTF-8: not a continuation byte.
+		}
+		return t.lookupValue(uint32(i), c2), 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return 0, 0
+		}
+		i := {{.StarterBlock}}[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return 0, 1 // Illegal UTF-8: not a continuation byte.
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = {{.Name}}Index[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return 0, 2 // Illegal UTF-8: not a continuation byte.
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = {{.Name}}Index[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return 0, 3 // Illegal UTF-8: not a continuation byte.
+		}
+		return t.lookupValue(uint32(i), c3), 4
+	}
+	// Illegal rune
+	return 0, 1
+}
+
+// lookup{{if eq .SourceType "string"}}String{{end}}Unsafe returns the trie value for the first UTF-8 encoding in s.
+// s must start with a full and valid UTF-8 encoded rune.
+func (t *{{.Name}}Trie) lookup{{if eq .SourceType "string"}}String{{end}}Unsafe(s {{.SourceType}}) {{.ValueType}} {
+	c0 := s[0]
+	if c0 < 0x80 { // is ASCII
+		return {{.ASCIIBlock}}[c0]
+	}
+	i := {{.StarterBlock}}[c0]
+	if c0 < 0xE0 { // 2-byte UTF-8
+		return t.lookupValue(uint32(i), s[1])
+	}
+	i = {{.Name}}Index[uint32(i)<<6+uint32(s[1])]
+	if c0 < 0xF0 { // 3-byte UTF-8
+		return t.lookupValue(uint32(i), s[2])
+	}
+	i = {{.Name}}Index[uint32(i)<<6+uint32(s[2])]
+	if c0 < 0xF8 { // 4-byte UTF-8
+		return t.lookupValue(uint32(i), s[3])
+	}
+	return 0
+}
+`