ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/encoding/simplifiedchinese/all.go

@@ -0,0 +1,12 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package simplifiedchinese
+
+import (
+	"golang.org/x/text/encoding"
+)
+
+// All is a list of all defined encodings in this package.
+var All = []encoding.Encoding{GB18030, GBK, HZGB2312}

data/ext/vendor/golang.org/x/text/encoding/simplifiedchinese/all_test.go

@@ -0,0 +1,143 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package simplifiedchinese
+
+import (
+	"strings"
+	"testing"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/internal"
+	"golang.org/x/text/encoding/internal/enctest"
+	"golang.org/x/text/transform"
+)
+
+func dec(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
+	return "Decode", e.NewDecoder(), nil
+}
+func enc(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
+	return "Encode", e.NewEncoder(), internal.ErrASCIIReplacement
+}
+
+func TestNonRepertoire(t *testing.T) {
+	// Pick n large enough to overflow the destination buffer of transform.String.
+	const n = 10000
+	testCases := []struct {
+		init      func(e encoding.Encoding) (string, transform.Transformer, error)
+		e         encoding.Encoding
+		src, want string
+	}{
+		{dec, GBK, "a\xfe\xfeb", "a\ufffdb"},
+		{dec, HZGB2312, "~{z~", "\ufffd"},
+
+		{enc, GBK, "갂", ""},
+		{enc, GBK, "a갂", "a"},
+		{enc, GBK, "\u4e02갂", "\x81@"},
+
+		{enc, HZGB2312, "갂", ""},
+		{enc, HZGB2312, "a갂", "a"},
+		{enc, HZGB2312, "\u6cf5갂", "~{1C~}"},
+
+		{dec, GB18030, "\x80", "€"},
+		{dec, GB18030, "\x81", "\ufffd"},
+		{dec, GB18030, "\x81\x20", "\ufffd "},
+		{dec, GB18030, "\xfe\xfe", "\ufffd"},
+		{dec, GB18030, "\xfe\xff", "\ufffd\ufffd"},
+		{dec, GB18030, "\xfe\x30", "\ufffd0"},
+		{dec, GB18030, "\xfe\x30\x30 ", "\ufffd00 "},
+		{dec, GB18030, "\xfe\x30\xff ", "\ufffd0\ufffd "},
+		{dec, GB18030, "\xfe\x30\x81\x21", "\ufffd0\ufffd!"},
+
+		{dec, GB18030, strings.Repeat("\xfe\x30", n), strings.Repeat("\ufffd0", n)},
+
+		{dec, HZGB2312, "~/", "\ufffd"},
+		{dec, HZGB2312, "~{a\x80", "\ufffd"},
+		{dec, HZGB2312, "~{a\x80", "\ufffd"},
+		{dec, HZGB2312, "~{" + strings.Repeat("z~", n), strings.Repeat("\ufffd", n)},
+		{dec, HZGB2312, "~{" + strings.Repeat("\xfe\x30", n), strings.Repeat("\ufffd", n*2)},
+	}
+	for _, tc := range testCases {
+		dir, tr, wantErr := tc.init(tc.e)
+
+		dst, _, err := transform.String(tr, tc.src)
+		if err != wantErr {
+			t.Errorf("%s %v(%q): got %v; want %v", dir, tc.e, tc.src, err, wantErr)
+		}
+		if got := string(dst); got != tc.want {
+			t.Errorf("%s %v(%q):\ngot  %q\nwant %q", dir, tc.e, tc.src, got, tc.want)
+		}
+	}
+}
+
+func TestBasics(t *testing.T) {
+	// The encoded forms can be verified by the iconv program:
+	// $ echo 月日は百代 | iconv -f UTF-8 -t SHIFT-JIS | xxd
+	testCases := []struct {
+		e         encoding.Encoding
+		encPrefix string
+		encoded   string
+		utf8      string
+	}{{
+		// "\u0081\u00de\u00df\u00e0\u00e1\u00e2\u00e3\uffff\U00010000" is a
+		// nonsense string that contains GB18030 encodable codepoints of which
+		// only U+00E0 and U+00E1 are GBK encodable.
+		//
+		// "A\u3000\u554a\u4e02\u4e90\u72dc\u7349\u02ca\u2588Z€" is a nonsense
+		// string that contains ASCII and GBK encodable codepoints from Levels
+		// 1-5 as well as the Euro sign.
+		//
+		// "A\u43f0\u4c32\U00027267\u3000\U0002910d\u79d4Z€" is a nonsense string
+		// that contains ASCII and Big5 encodable codepoints from the Basic
+		// Multilingual Plane and the Supplementary Ideographic Plane as well as
+		// the Euro sign.
+		//
+		// "花间一壶酒，独酌无相亲。" (simplified) and
+		// "花間一壺酒，獨酌無相親。" (traditional)
+		// are from the 8th century poem "Yuè Xià Dú Zhuó".
+		e: GB18030,
+		encoded: "\x81\x30\x81\x31\x81\x30\x89\x37\x81\x30\x89\x38\xa8\xa4\xa8\xa2" +
+			"\x81\x30\x89\x39\x81\x30\x8a\x30\x84\x31\xa4\x39\x90\x30\x81\x30",
+		utf8: "\u0081\u00de\u00df\u00e0\u00e1\u00e2\u00e3\uffff\U00010000",
+	}, {
+		e: GB18030,
+		encoded: "\xbb\xa8\xbc\xe4\xd2\xbb\xba\xf8\xbe\xc6\xa3\xac\xb6\xc0\xd7\xc3" +
+			"\xce\xde\xcf\xe0\xc7\xd7\xa1\xa3",
+		utf8: "花间一壶酒，独酌无相亲。",
+	}, {
+		e:       GBK,
+		encoded: "A\xa1\xa1\xb0\xa1\x81\x40\x81\x80\xaa\x40\xaa\x80\xa8\x40\xa8\x80Z\x80",
+		utf8:    "A\u3000\u554a\u4e02\u4e90\u72dc\u7349\u02ca\u2588Z€",
+	}, {
+		e: GBK,
+		encoded: "\xbb\xa8\xbc\xe4\xd2\xbb\xba\xf8\xbe\xc6\xa3\xac\xb6\xc0\xd7\xc3" +
+			"\xce\xde\xcf\xe0\xc7\xd7\xa1\xa3",
+		utf8: "花间一壶酒，独酌无相亲。",
+	}, {
+		e:       HZGB2312,
+		encoded: "A~{\x21\x21~~\x30\x21~}Z~~",
+		utf8:    "A\u3000~\u554aZ~",
+	}, {
+		e:         HZGB2312,
+		encPrefix: "~{",
+		encoded:   ";(<dR;:x>F#,6@WCN^O`GW!#",
+		utf8:      "花间一壶酒，独酌无相亲。",
+	}}
+
+	for _, tc := range testCases {
+		enctest.TestEncoding(t, tc.e, tc.encoded, tc.utf8, tc.encPrefix, "")
+	}
+}
+
+func TestFiles(t *testing.T) {
+	enctest.TestFile(t, GB18030)
+	enctest.TestFile(t, GBK)
+	enctest.TestFile(t, HZGB2312)
+}
+
+func BenchmarkEncoding(b *testing.B) {
+	enctest.Benchmark(b, GB18030)
+	enctest.Benchmark(b, GBK)
+	enctest.Benchmark(b, HZGB2312)
+}

data/ext/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go

@@ -0,0 +1,269 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package simplifiedchinese
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/internal"
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/transform"
+)
+
+var (
+	// GB18030 is the GB18030 encoding.
+	GB18030 encoding.Encoding = &gbk18030
+	// GBK is the GBK encoding. It encodes an extension of the GB2312 character set
+	// and is also known as Code Page 936.
+	GBK encoding.Encoding = &gbk
+)
+
+var gbk = internal.Encoding{
+	&internal.SimpleEncoding{
+		gbkDecoder{gb18030: false},
+		gbkEncoder{gb18030: false},
+	},
+	"GBK",
+	identifier.GBK,
+}
+
+var gbk18030 = internal.Encoding{
+	&internal.SimpleEncoding{
+		gbkDecoder{gb18030: true},
+		gbkEncoder{gb18030: true},
+	},
+	"GB18030",
+	identifier.GB18030,
+}
+
+type gbkDecoder struct {
+	transform.NopResetter
+	gb18030 bool
+}
+
+func (d gbkDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	r, size := rune(0), 0
+loop:
+	for ; nSrc < len(src); nSrc += size {
+		switch c0 := src[nSrc]; {
+		case c0 < utf8.RuneSelf:
+			r, size = rune(c0), 1
+
+		// Microsoft's Code Page 936 extends GBK 1.0 to encode the euro sign U+20AC
+		// as 0x80. The HTML5 specification at http://encoding.spec.whatwg.org/#gbk
+		// says to treat "gbk" as Code Page 936.
+		case c0 == 0x80:
+			r, size = '€', 1
+
+		case c0 < 0xff:
+			if nSrc+1 >= len(src) {
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				goto write
+			}
+			c1 := src[nSrc+1]
+			switch {
+			case 0x40 <= c1 && c1 < 0x7f:
+				c1 -= 0x40
+			case 0x80 <= c1 && c1 < 0xff:
+				c1 -= 0x41
+			case d.gb18030 && 0x30 <= c1 && c1 < 0x40:
+				if nSrc+3 >= len(src) {
+					if !atEOF {
+						err = transform.ErrShortSrc
+						break loop
+					}
+					// The second byte here is always ASCII, so we can set size
+					// to 1 in all cases.
+					r, size = utf8.RuneError, 1
+					goto write
+				}
+				c2 := src[nSrc+2]
+				if c2 < 0x81 || 0xff <= c2 {
+					r, size = utf8.RuneError, 1
+					goto write
+				}
+				c3 := src[nSrc+3]
+				if c3 < 0x30 || 0x3a <= c3 {
+					r, size = utf8.RuneError, 1
+					goto write
+				}
+				size = 4
+				r = ((rune(c0-0x81)*10+rune(c1-0x30))*126+rune(c2-0x81))*10 + rune(c3-0x30)
+				if r < 39420 {
+					i, j := 0, len(gb18030)
+					for i < j {
+						h := i + (j-i)/2
+						if r >= rune(gb18030[h][0]) {
+							i = h + 1
+						} else {
+							j = h
+						}
+					}
+					dec := &gb18030[i-1]
+					r += rune(dec[1]) - rune(dec[0])
+					goto write
+				}
+				r -= 189000
+				if 0 <= r && r < 0x100000 {
+					r += 0x10000
+				} else {
+					r, size = utf8.RuneError, 1
+				}
+				goto write
+			default:
+				r, size = utf8.RuneError, 1
+				goto write
+			}
+			r, size = '\ufffd', 2
+			if i := int(c0-0x81)*190 + int(c1); i < len(decode) {
+				r = rune(decode[i])
+				if r == 0 {
+					r = '\ufffd'
+				}
+			}
+
+		default:
+			r, size = utf8.RuneError, 1
+		}
+
+	write:
+		if nDst+utf8.RuneLen(r) > len(dst) {
+			err = transform.ErrShortDst
+			break loop
+		}
+		nDst += utf8.EncodeRune(dst[nDst:], r)
+	}
+	return nDst, nSrc, err
+}
+
+type gbkEncoder struct {
+	transform.NopResetter
+	gb18030 bool
+}
+
+func (e gbkEncoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	r, r2, size := rune(0), rune(0), 0
+	for ; nSrc < len(src); nSrc += size {
+		r = rune(src[nSrc])
+
+		// Decode a 1-byte rune.
+		if r < utf8.RuneSelf {
+			size = 1
+
+		} else {
+			// Decode a multi-byte rune.
+			r, size = utf8.DecodeRune(src[nSrc:])
+			if size == 1 {
+				// All valid runes of size 1 (those below utf8.RuneSelf) were
+				// handled above. We have invalid UTF-8 or we haven't seen the
+				// full character yet.
+				if !atEOF && !utf8.FullRune(src[nSrc:]) {
+					err = transform.ErrShortSrc
+					break
+				}
+			}
+
+			// func init checks that the switch covers all tables.
+			switch {
+			case encode0Low <= r && r < encode0High:
+				if r2 = rune(encode0[r-encode0Low]); r2 != 0 {
+					goto write2
+				}
+			case encode1Low <= r && r < encode1High:
+				// Microsoft's Code Page 936 extends GBK 1.0 to encode the euro sign U+20AC
+				// as 0x80. The HTML5 specification at http://encoding.spec.whatwg.org/#gbk
+				// says to treat "gbk" as Code Page 936.
+				if r == '€' {
+					r = 0x80
+					goto write1
+				}
+				if r2 = rune(encode1[r-encode1Low]); r2 != 0 {
+					goto write2
+				}
+			case encode2Low <= r && r < encode2High:
+				if r2 = rune(encode2[r-encode2Low]); r2 != 0 {
+					goto write2
+				}
+			case encode3Low <= r && r < encode3High:
+				if r2 = rune(encode3[r-encode3Low]); r2 != 0 {
+					goto write2
+				}
+			case encode4Low <= r && r < encode4High:
+				if r2 = rune(encode4[r-encode4Low]); r2 != 0 {
+					goto write2
+				}
+			}
+
+			if e.gb18030 {
+				if r < 0x10000 {
+					i, j := 0, len(gb18030)
+					for i < j {
+						h := i + (j-i)/2
+						if r >= rune(gb18030[h][1]) {
+							i = h + 1
+						} else {
+							j = h
+						}
+					}
+					dec := &gb18030[i-1]
+					r += rune(dec[0]) - rune(dec[1])
+					goto write4
+				} else if r < 0x110000 {
+					r += 189000 - 0x10000
+					goto write4
+				}
+			}
+			err = internal.ErrASCIIReplacement
+			break
+		}
+
+	write1:
+		if nDst >= len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		dst[nDst] = uint8(r)
+		nDst++
+		continue
+
+	write2:
+		if nDst+2 > len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		dst[nDst+0] = uint8(r2 >> 8)
+		dst[nDst+1] = uint8(r2)
+		nDst += 2
+		continue
+
+	write4:
+		if nDst+4 > len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		dst[nDst+3] = uint8(r%10 + 0x30)
+		r /= 10
+		dst[nDst+2] = uint8(r%126 + 0x81)
+		r /= 126
+		dst[nDst+1] = uint8(r%10 + 0x30)
+		r /= 10
+		dst[nDst+0] = uint8(r + 0x81)
+		nDst += 4
+		continue
+	}
+	return nDst, nSrc, err
+}
+
+func init() {
+	// Check that the hard-coded encode switch covers all tables.
+	if numEncodeTables != 5 {
+		panic("bad numEncodeTables")
+	}
+}

data/ext/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go

@@ -0,0 +1,245 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package simplifiedchinese
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/internal"
+	"golang.org/x/text/encoding/internal/identifier"
+	"golang.org/x/text/transform"
+)
+
+// HZGB2312 is the HZ-GB2312 encoding.
+var HZGB2312 encoding.Encoding = &hzGB2312
+
+var hzGB2312 = internal.Encoding{
+	internal.FuncEncoding{hzGB2312NewDecoder, hzGB2312NewEncoder},
+	"HZ-GB2312",
+	identifier.HZGB2312,
+}
+
+func hzGB2312NewDecoder() transform.Transformer {
+	return new(hzGB2312Decoder)
+}
+
+func hzGB2312NewEncoder() transform.Transformer {
+	return new(hzGB2312Encoder)
+}
+
+const (
+	asciiState = iota
+	gbState
+)
+
+type hzGB2312Decoder int
+
+func (d *hzGB2312Decoder) Reset() {
+	*d = asciiState
+}
+
+func (d *hzGB2312Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	r, size := rune(0), 0
+loop:
+	for ; nSrc < len(src); nSrc += size {
+		c0 := src[nSrc]
+		if c0 >= utf8.RuneSelf {
+			r, size = utf8.RuneError, 1
+			goto write
+		}
+
+		if c0 == '~' {
+			if nSrc+1 >= len(src) {
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r = utf8.RuneError
+				goto write
+			}
+			size = 2
+			switch src[nSrc+1] {
+			case '{':
+				*d = gbState
+				continue
+			case '}':
+				*d = asciiState
+				continue
+			case '~':
+				if nDst >= len(dst) {
+					err = transform.ErrShortDst
+					break loop
+				}
+				dst[nDst] = '~'
+				nDst++
+				continue
+			case '\n':
+				continue
+			default:
+				r = utf8.RuneError
+				goto write
+			}
+		}
+
+		if *d == asciiState {
+			r, size = rune(c0), 1
+		} else {
+			if nSrc+1 >= len(src) {
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				goto write
+			}
+			size = 2
+			c1 := src[nSrc+1]
+			if c0 < 0x21 || 0x7e <= c0 || c1 < 0x21 || 0x7f <= c1 {
+				// error
+			} else if i := int(c0-0x01)*190 + int(c1+0x3f); i < len(decode) {
+				r = rune(decode[i])
+				if r != 0 {
+					goto write
+				}
+			}
+			if c1 > utf8.RuneSelf {
+				// Be consistent and always treat non-ASCII as a single error.
+				size = 1
+			}
+			r = utf8.RuneError
+		}
+
+	write:
+		if nDst+utf8.RuneLen(r) > len(dst) {
+			err = transform.ErrShortDst
+			break loop
+		}
+		nDst += utf8.EncodeRune(dst[nDst:], r)
+	}
+	return nDst, nSrc, err
+}
+
+type hzGB2312Encoder int
+
+func (d *hzGB2312Encoder) Reset() {
+	*d = asciiState
+}
+
+func (e *hzGB2312Encoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	r, size := rune(0), 0
+	for ; nSrc < len(src); nSrc += size {
+		r = rune(src[nSrc])
+
+		// Decode a 1-byte rune.
+		if r < utf8.RuneSelf {
+			size = 1
+			if r == '~' {
+				if nDst+2 > len(dst) {
+					err = transform.ErrShortDst
+					break
+				}
+				dst[nDst+0] = '~'
+				dst[nDst+1] = '~'
+				nDst += 2
+				continue
+			} else if *e != asciiState {
+				if nDst+3 > len(dst) {
+					err = transform.ErrShortDst
+					break
+				}
+				*e = asciiState
+				dst[nDst+0] = '~'
+				dst[nDst+1] = '}'
+				nDst += 2
+			} else if nDst >= len(dst) {
+				err = transform.ErrShortDst
+				break
+			}
+			dst[nDst] = uint8(r)
+			nDst += 1
+			continue
+
+		}
+
+		// Decode a multi-byte rune.
+		r, size = utf8.DecodeRune(src[nSrc:])
+		if size == 1 {
+			// All valid runes of size 1 (those below utf8.RuneSelf) were
+			// handled above. We have invalid UTF-8 or we haven't seen the
+			// full character yet.
+			if !atEOF && !utf8.FullRune(src[nSrc:]) {
+				err = transform.ErrShortSrc
+				break
+			}
+		}
+
+		// func init checks that the switch covers all tables.
+		switch {
+		case encode0Low <= r && r < encode0High:
+			if r = rune(encode0[r-encode0Low]); r != 0 {
+				goto writeGB
+			}
+		case encode1Low <= r && r < encode1High:
+			if r = rune(encode1[r-encode1Low]); r != 0 {
+				goto writeGB
+			}
+		case encode2Low <= r && r < encode2High:
+			if r = rune(encode2[r-encode2Low]); r != 0 {
+				goto writeGB
+			}
+		case encode3Low <= r && r < encode3High:
+			if r = rune(encode3[r-encode3Low]); r != 0 {
+				goto writeGB
+			}
+		case encode4Low <= r && r < encode4High:
+			if r = rune(encode4[r-encode4Low]); r != 0 {
+				goto writeGB
+			}
+		}
+
+	terminateInASCIIState:
+		// Switch back to ASCII state in case of error so that an ASCII
+		// replacement character can be written in the correct state.
+		if *e != asciiState {
+			if nDst+2 > len(dst) {
+				err = transform.ErrShortDst
+				break
+			}
+			dst[nDst+0] = '~'
+			dst[nDst+1] = '}'
+			nDst += 2
+		}
+		err = internal.ErrASCIIReplacement
+		break
+
+	writeGB:
+		c0 := uint8(r>>8) - 0x80
+		c1 := uint8(r) - 0x80
+		if c0 < 0x21 || 0x7e <= c0 || c1 < 0x21 || 0x7f <= c1 {
+			goto terminateInASCIIState
+		}
+		if *e == asciiState {
+			if nDst+4 > len(dst) {
+				err = transform.ErrShortDst
+				break
+			}
+			*e = gbState
+			dst[nDst+0] = '~'
+			dst[nDst+1] = '{'
+			nDst += 2
+		} else if nDst+2 > len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		dst[nDst+0] = c0
+		dst[nDst+1] = c1
+		nDst += 2
+		continue
+	}
+	// TODO: should one always terminate in ASCII state to make it safe to
+	// concatenate two HZ-GB2312-encoded strings?
+	return nDst, nSrc, err
+}