ires 0.1.3 → 0.1.4

data/ext/vendor/golang.org/x/text/runes/example_test.go

@@ -0,0 +1,60 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runes_test
+
+import (
+	"fmt"
+	"unicode"
+
+	"golang.org/x/text/runes"
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
+	"golang.org/x/text/width"
+)
+
+func ExampleRemove() {
+	t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
+	s, _, _ := transform.String(t, "résumé")
+	fmt.Println(s)
+
+	// Output:
+	// resume
+}
+
+func ExampleMap() {
+	replaceHyphens := runes.Map(func(r rune) rune {
+		if unicode.Is(unicode.Hyphen, r) {
+			return '|'
+		}
+		return r
+	})
+	s, _, _ := transform.String(replaceHyphens, "a-b‐c⸗d﹣e")
+	fmt.Println(s)
+
+	// Output:
+	// a|b|c|d|e
+}
+
+func ExampleIn() {
+	// Convert Latin characters to their canonical form, while keeping other
+	// width distinctions.
+	t := runes.If(runes.In(unicode.Latin), width.Fold, nil)
+	s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / アルアノリウ ｔｅｃｈ")
+	fmt.Println(s)
+
+	// Output:
+	// ｱﾙｱﾉﾘｳ tech / アルアノリウ tech
+}
+
+func ExampleIf() {
+	// Widen everything but ASCII.
+	isASCII := func(r rune) bool { return r <= unicode.MaxASCII }
+	t := runes.If(runes.Predicate(isASCII), nil, width.Widen)
+	s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / 中國 / 5₩")
+	fmt.Println(s)
+
+	// Output:
+	// アルアノリウ tech / 中國 / 5￦
+}

data/ext/vendor/golang.org/x/text/runes/runes.go

@@ -0,0 +1,355 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package runes provide transforms for UTF-8 encoded text.
+package runes // import "golang.org/x/text/runes"
+
+import (
+	"unicode"
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+// A Set is a collection of runes.
+type Set interface {
+	// Contains returns true if r is contained in the set.
+	Contains(r rune) bool
+}
+
+type setFunc func(rune) bool
+
+func (s setFunc) Contains(r rune) bool {
+	return s(r)
+}
+
+// Note: using funcs here instead of wrapping types result in cleaner
+// documentation and a smaller API.
+
+// In creates a Set with a Contains method that returns true for all runes in
+// the given RangeTable.
+func In(rt *unicode.RangeTable) Set {
+	return setFunc(func(r rune) bool { return unicode.Is(rt, r) })
+}
+
+// In creates a Set with a Contains method that returns true for all runes not
+// in the given RangeTable.
+func NotIn(rt *unicode.RangeTable) Set {
+	return setFunc(func(r rune) bool { return !unicode.Is(rt, r) })
+}
+
+// Predicate creates a Set with a Contains method that returns f(r).
+func Predicate(f func(rune) bool) Set {
+	return setFunc(f)
+}
+
+// Transformer implements the transform.Transformer interface.
+type Transformer struct {
+	t transform.SpanningTransformer
+}
+
+func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	return t.t.Transform(dst, src, atEOF)
+}
+
+func (t Transformer) Span(b []byte, atEOF bool) (n int, err error) {
+	return t.t.Span(b, atEOF)
+}
+
+func (t Transformer) Reset() { t.t.Reset() }
+
+// Bytes returns a new byte slice with the result of converting b using t.  It
+// calls Reset on t. It returns nil if any error was found. This can only happen
+// if an error-producing Transformer is passed to If.
+func (t Transformer) Bytes(b []byte) []byte {
+	b, _, err := transform.Bytes(t, b)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+// String returns a string with the result of converting s using t. It calls
+// Reset on t. It returns the empty string if any error was found. This can only
+// happen if an error-producing Transformer is passed to If.
+func (t Transformer) String(s string) string {
+	s, _, err := transform.String(t, s)
+	if err != nil {
+		return ""
+	}
+	return s
+}
+
+// TODO:
+// - Copy: copying strings and bytes in whole-rune units.
+// - Validation (maybe)
+// - Well-formed-ness (maybe)
+
+const runeErrorString = string(utf8.RuneError)
+
+// Remove returns a Transformer that removes runes r for which s.Contains(r).
+// Illegal input bytes are replaced by RuneError before being passed to f.
+func Remove(s Set) Transformer {
+	if f, ok := s.(setFunc); ok {
+		// This little trick cuts the running time of BenchmarkRemove for sets
+		// created by Predicate roughly in half.
+		// TODO: special-case RangeTables as well.
+		return Transformer{remove(f)}
+	}
+	return Transformer{remove(s.Contains)}
+}
+
+// TODO: remove transform.RemoveFunc.
+
+type remove func(r rune) bool
+
+func (remove) Reset() {}
+
+// Span implements transform.Spanner.
+func (t remove) Span(src []byte, atEOF bool) (n int, err error) {
+	for r, size := rune(0), 0; n < len(src); {
+		if r = rune(src[n]); r < utf8.RuneSelf {
+			size = 1
+		} else if r, size = utf8.DecodeRune(src[n:]); size == 1 {
+			// Invalid rune.
+			if !atEOF && !utf8.FullRune(src[n:]) {
+				err = transform.ErrShortSrc
+			} else {
+				err = transform.ErrEndOfSpan
+			}
+			break
+		}
+		if t(r) {
+			err = transform.ErrEndOfSpan
+			break
+		}
+		n += size
+	}
+	return
+}
+
+// Transform implements transform.Transformer.
+func (t remove) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for r, size := rune(0), 0; nSrc < len(src); {
+		if r = rune(src[nSrc]); r < utf8.RuneSelf {
+			size = 1
+		} else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 {
+			// Invalid rune.
+			if !atEOF && !utf8.FullRune(src[nSrc:]) {
+				err = transform.ErrShortSrc
+				break
+			}
+			// We replace illegal bytes with RuneError. Not doing so might
+			// otherwise turn a sequence of invalid UTF-8 into valid UTF-8.
+			// The resulting byte sequence may subsequently contain runes
+			// for which t(r) is true that were passed unnoticed.
+			if !t(utf8.RuneError) {
+				if nDst+3 > len(dst) {
+					err = transform.ErrShortDst
+					break
+				}
+				dst[nDst+0] = runeErrorString[0]
+				dst[nDst+1] = runeErrorString[1]
+				dst[nDst+2] = runeErrorString[2]
+				nDst += 3
+			}
+			nSrc++
+			continue
+		}
+		if t(r) {
+			nSrc += size
+			continue
+		}
+		if nDst+size > len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		for i := 0; i < size; i++ {
+			dst[nDst] = src[nSrc]
+			nDst++
+			nSrc++
+		}
+	}
+	return
+}
+
+// Map returns a Transformer that maps the runes in the input using the given
+// mapping. Illegal bytes in the input are converted to utf8.RuneError before
+// being passed to the mapping func.
+func Map(mapping func(rune) rune) Transformer {
+	return Transformer{mapper(mapping)}
+}
+
+type mapper func(rune) rune
+
+func (mapper) Reset() {}
+
+// Span implements transform.Spanner.
+func (t mapper) Span(src []byte, atEOF bool) (n int, err error) {
+	for r, size := rune(0), 0; n < len(src); n += size {
+		if r = rune(src[n]); r < utf8.RuneSelf {
+			size = 1
+		} else if r, size = utf8.DecodeRune(src[n:]); size == 1 {
+			// Invalid rune.
+			if !atEOF && !utf8.FullRune(src[n:]) {
+				err = transform.ErrShortSrc
+			} else {
+				err = transform.ErrEndOfSpan
+			}
+			break
+		}
+		if t(r) != r {
+			err = transform.ErrEndOfSpan
+			break
+		}
+	}
+	return n, err
+}
+
+// Transform implements transform.Transformer.
+func (t mapper) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	var replacement rune
+	var b [utf8.UTFMax]byte
+
+	for r, size := rune(0), 0; nSrc < len(src); {
+		if r = rune(src[nSrc]); r < utf8.RuneSelf {
+			if replacement = t(r); replacement < utf8.RuneSelf {
+				if nDst == len(dst) {
+					err = transform.ErrShortDst
+					break
+				}
+				dst[nDst] = byte(replacement)
+				nDst++
+				nSrc++
+				continue
+			}
+			size = 1
+		} else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 {
+			// Invalid rune.
+			if !atEOF && !utf8.FullRune(src[nSrc:]) {
+				err = transform.ErrShortSrc
+				break
+			}
+
+			if replacement = t(utf8.RuneError); replacement == utf8.RuneError {
+				if nDst+3 > len(dst) {
+					err = transform.ErrShortDst
+					break
+				}
+				dst[nDst+0] = runeErrorString[0]
+				dst[nDst+1] = runeErrorString[1]
+				dst[nDst+2] = runeErrorString[2]
+				nDst += 3
+				nSrc++
+				continue
+			}
+		} else if replacement = t(r); replacement == r {
+			if nDst+size > len(dst) {
+				err = transform.ErrShortDst
+				break
+			}
+			for i := 0; i < size; i++ {
+				dst[nDst] = src[nSrc]
+				nDst++
+				nSrc++
+			}
+			continue
+		}
+
+		n := utf8.EncodeRune(b[:], replacement)
+
+		if nDst+n > len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		for i := 0; i < n; i++ {
+			dst[nDst] = b[i]
+			nDst++
+		}
+		nSrc += size
+	}
+	return
+}
+
+// ReplaceIllFormed returns a transformer that replaces all input bytes that are
+// not part of a well-formed UTF-8 code sequence with utf8.RuneError.
+func ReplaceIllFormed() Transformer {
+	return Transformer{&replaceIllFormed{}}
+}
+
+type replaceIllFormed struct{ transform.NopResetter }
+
+func (t replaceIllFormed) Span(src []byte, atEOF bool) (n int, err error) {
+	for n < len(src) {
+		// ASCII fast path.
+		if src[n] < utf8.RuneSelf {
+			n++
+			continue
+		}
+
+		r, size := utf8.DecodeRune(src[n:])
+
+		// Look for a valid non-ASCII rune.
+		if r != utf8.RuneError || size != 1 {
+			n += size
+			continue
+		}
+
+		// Look for short source data.
+		if !atEOF && !utf8.FullRune(src[n:]) {
+			err = transform.ErrShortSrc
+			break
+		}
+
+		// We have an invalid rune.
+		err = transform.ErrEndOfSpan
+		break
+	}
+	return n, err
+}
+
+func (t replaceIllFormed) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for nSrc < len(src) {
+		// ASCII fast path.
+		if r := src[nSrc]; r < utf8.RuneSelf {
+			if nDst == len(dst) {
+				err = transform.ErrShortDst
+				break
+			}
+			dst[nDst] = r
+			nDst++
+			nSrc++
+			continue
+		}
+
+		// Look for a valid non-ASCII rune.
+		if _, size := utf8.DecodeRune(src[nSrc:]); size != 1 {
+			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
+				err = transform.ErrShortDst
+				break
+			}
+			nDst += size
+			nSrc += size
+			continue
+		}
+
+		// Look for short source data.
+		if !atEOF && !utf8.FullRune(src[nSrc:]) {
+			err = transform.ErrShortSrc
+			break
+		}
+
+		// We have an invalid rune.
+		if nDst+3 > len(dst) {
+			err = transform.ErrShortDst
+			break
+		}
+		dst[nDst+0] = runeErrorString[0]
+		dst[nDst+1] = runeErrorString[1]
+		dst[nDst+2] = runeErrorString[2]
+		nDst += 3
+		nSrc++
+	}
+	return nDst, nSrc, err
+}

data/ext/vendor/golang.org/x/text/runes/runes_test.go

@@ -0,0 +1,664 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runes
+
+import (
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	"golang.org/x/text/internal/testtext"
+	"golang.org/x/text/transform"
+)
+
+type transformTest struct {
+	desc    string
+	szDst   int
+	atEOF   bool
+	repl    string
+	in      string
+	out     string // result string of first call to Transform
+	outFull string // transform of entire input string
+	err     error
+	errSpan error
+	nSpan   int
+
+	t transform.SpanningTransformer
+}
+
+const large = 10240
+
+func (tt *transformTest) check(t *testing.T, i int) {
+	if tt.t == nil {
+		return
+	}
+	dst := make([]byte, tt.szDst)
+	src := []byte(tt.in)
+	nDst, nSrc, err := tt.t.Transform(dst, src, tt.atEOF)
+	if err != tt.err {
+		t.Errorf("%d:%s:error: got %v; want %v", i, tt.desc, err, tt.err)
+	}
+	if got := string(dst[:nDst]); got != tt.out {
+		t.Errorf("%d:%s:out: got %q; want %q", i, tt.desc, got, tt.out)
+	}
+
+	// Calls tt.t.Transform for the remainder of the input. We use this to test
+	// the nSrc return value.
+	out := make([]byte, large)
+	n := copy(out, dst[:nDst])
+	nDst, _, _ = tt.t.Transform(out[n:], src[nSrc:], true)
+	if got, want := string(out[:n+nDst]), tt.outFull; got != want {
+		t.Errorf("%d:%s:outFull: got %q; want %q", i, tt.desc, got, want)
+	}
+
+	tt.t.Reset()
+	p := 0
+	for ; p < len(tt.in) && p < len(tt.outFull) && tt.in[p] == tt.outFull[p]; p++ {
+	}
+	if tt.nSpan != 0 {
+		p = tt.nSpan
+	}
+	if n, err = tt.t.Span([]byte(tt.in), tt.atEOF); n != p || err != tt.errSpan {
+		t.Errorf("%d:%s:span: got %d, %v; want %d, %v", i, tt.desc, n, err, p, tt.errSpan)
+	}
+}
+
+func idem(r rune) rune { return r }
+
+func TestMap(t *testing.T) {
+	runes := []rune{'a', 'ç', '中', '\U00012345', 'a'}
+	// Default mapper used for this test.
+	rotate := Map(func(r rune) rune {
+		for i, m := range runes {
+			if m == r {
+				return runes[i+1]
+			}
+		}
+		return r
+	})
+
+	for i, tt := range []transformTest{{
+		desc:    "empty",
+		szDst:   large,
+		atEOF:   true,
+		in:      "",
+		out:     "",
+		outFull: "",
+		t:       rotate,
+	}, {
+		desc:    "no change",
+		szDst:   1,
+		atEOF:   true,
+		in:      "b",
+		out:     "b",
+		outFull: "b",
+		t:       rotate,
+	}, {
+		desc:    "short dst",
+		szDst:   2,
+		atEOF:   true,
+		in:      "aaaa",
+		out:     "ç",
+		outFull: "çççç",
+		err:     transform.ErrShortDst,
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "short dst ascii, no change",
+		szDst:   2,
+		atEOF:   true,
+		in:      "bbb",
+		out:     "bb",
+		outFull: "bbb",
+		err:     transform.ErrShortDst,
+		t:       rotate,
+	}, {
+		desc:    "short dst writing error",
+		szDst:   2,
+		atEOF:   false,
+		in:      "a\x80",
+		out:     "ç",
+		outFull: "ç\ufffd",
+		err:     transform.ErrShortDst,
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "short dst writing incomplete rune",
+		szDst:   2,
+		atEOF:   true,
+		in:      "a\xc0",
+		out:     "ç",
+		outFull: "ç\ufffd",
+		err:     transform.ErrShortDst,
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "short dst, longer",
+		szDst:   5,
+		atEOF:   true,
+		in:      "Hellø",
+		out:     "Hell",
+		outFull: "Hellø",
+		err:     transform.ErrShortDst,
+		t:       rotate,
+	}, {
+		desc:    "short dst, single",
+		szDst:   1,
+		atEOF:   false,
+		in:      "ø",
+		out:     "",
+		outFull: "ø",
+		err:     transform.ErrShortDst,
+		t:       Map(idem),
+	}, {
+		desc:    "short dst, longer, writing error",
+		szDst:   8,
+		atEOF:   false,
+		in:      "\x80Hello\x80",
+		out:     "\ufffdHello",
+		outFull: "\ufffdHello\ufffd",
+		err:     transform.ErrShortDst,
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "short src",
+		szDst:   2,
+		atEOF:   false,
+		in:      "a\xc2",
+		out:     "ç",
+		outFull: "ç\ufffd",
+		err:     transform.ErrShortSrc,
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "invalid input, atEOF",
+		szDst:   large,
+		atEOF:   true,
+		in:      "\x80",
+		out:     "\ufffd",
+		outFull: "\ufffd",
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "invalid input, !atEOF",
+		szDst:   large,
+		atEOF:   false,
+		in:      "\x80",
+		out:     "\ufffd",
+		outFull: "\ufffd",
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "incomplete rune !atEOF",
+		szDst:   large,
+		atEOF:   false,
+		in:      "\xc2",
+		out:     "",
+		outFull: "\ufffd",
+		err:     transform.ErrShortSrc,
+		errSpan: transform.ErrShortSrc,
+		t:       rotate,
+	}, {
+		desc:    "invalid input, incomplete rune atEOF",
+		szDst:   large,
+		atEOF:   true,
+		in:      "\xc2",
+		out:     "\ufffd",
+		outFull: "\ufffd",
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "misc correct",
+		szDst:   large,
+		atEOF:   true,
+		in:      "a\U00012345 ç!",
+		out:     "ça 中!",
+		outFull: "ça 中!",
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "misc correct and invalid",
+		szDst:   large,
+		atEOF:   true,
+		in:      "Hello\x80 w\x80orl\xc0d!\xc0",
+		out:     "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+		outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "misc correct and invalid, short src",
+		szDst:   large,
+		atEOF:   false,
+		in:      "Hello\x80 w\x80orl\xc0d!\xc2",
+		out:     "Hello\ufffd w\ufffdorl\ufffdd!",
+		outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+		err:     transform.ErrShortSrc,
+		errSpan: transform.ErrEndOfSpan,
+		t:       rotate,
+	}, {
+		desc:    "misc correct and invalid, short src, replacing RuneError",
+		szDst:   large,
+		atEOF:   false,
+		in:      "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc2",
+		out:     "Hel?lo? w?orl?d!",
+		outFull: "Hel?lo? w?orl?d!?",
+		errSpan: transform.ErrEndOfSpan,
+		err:     transform.ErrShortSrc,
+		t: Map(func(r rune) rune {
+			if r == utf8.RuneError {
+				return '?'
+			}
+			return r
+		}),
+	}} {
+		tt.check(t, i)
+	}
+}
+
+func TestRemove(t *testing.T) {
+	remove := Remove(Predicate(func(r rune) bool {
+		return strings.ContainsRune("aeiou\u0300\uFF24\U00012345", r)
+	}))
+
+	for i, tt := range []transformTest{
+		0: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "",
+			out:     "",
+			outFull: "",
+			t:       remove,
+		},
+		1: {
+			szDst:   0,
+			atEOF:   true,
+			in:      "aaaa",
+			out:     "",
+			outFull: "",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		2: {
+			szDst:   1,
+			atEOF:   true,
+			in:      "aaaa",
+			out:     "",
+			outFull: "",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		3: {
+			szDst:   1,
+			atEOF:   true,
+			in:      "baaaa",
+			out:     "b",
+			outFull: "b",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		4: {
+			szDst:   2,
+			atEOF:   true,
+			in:      "açaaa",
+			out:     "ç",
+			outFull: "ç",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		5: {
+			szDst:   2,
+			atEOF:   true,
+			in:      "aaaç",
+			out:     "ç",
+			outFull: "ç",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		6: {
+			szDst:   2,
+			atEOF:   false,
+			in:      "a\x80",
+			out:     "",
+			outFull: "\ufffd",
+			err:     transform.ErrShortDst,
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		7: {
+			szDst:   1,
+			atEOF:   true,
+			in:      "a\xc0",
+			out:     "",
+			outFull: "\ufffd",
+			err:     transform.ErrShortDst,
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		8: {
+			szDst:   1,
+			atEOF:   false,
+			in:      "a\xc2",
+			out:     "",
+			outFull: "\ufffd",
+			err:     transform.ErrShortSrc,
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		9: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "\x80",
+			out:     "\ufffd",
+			outFull: "\ufffd",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		10: {
+			szDst:   large,
+			atEOF:   false,
+			in:      "\x80",
+			out:     "\ufffd",
+			outFull: "\ufffd",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		11: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "\xc2",
+			out:     "\ufffd",
+			outFull: "\ufffd",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		12: {
+			szDst:   large,
+			atEOF:   false,
+			in:      "\xc2",
+			out:     "",
+			outFull: "\ufffd",
+			err:     transform.ErrShortSrc,
+			errSpan: transform.ErrShortSrc,
+			t:       remove,
+		},
+		13: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "Hello \U00012345world!",
+			out:     "Hll wrld!",
+			outFull: "Hll wrld!",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		14: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "Hello\x80 w\x80orl\xc0d!\xc0",
+			out:     "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
+			outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		15: {
+			szDst:   large,
+			atEOF:   false,
+			in:      "Hello\x80 w\x80orl\xc0d!\xc2",
+			out:     "Hll\ufffd w\ufffdrl\ufffdd!",
+			outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
+			err:     transform.ErrShortSrc,
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		16: {
+			szDst:   large,
+			atEOF:   false,
+			in:      "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc2",
+			out:     "Hello world!",
+			outFull: "Hello world!",
+			err:     transform.ErrShortSrc,
+			errSpan: transform.ErrEndOfSpan,
+			t:       Remove(Predicate(func(r rune) bool { return r == utf8.RuneError })),
+		},
+		17: {
+			szDst:   4,
+			atEOF:   true,
+			in:      "Hellø",
+			out:     "Hll",
+			outFull: "Hllø",
+			err:     transform.ErrShortDst,
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		18: {
+			szDst:   4,
+			atEOF:   false,
+			in:      "Hellø",
+			out:     "Hll",
+			outFull: "Hllø",
+			err:     transform.ErrShortDst,
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		19: {
+			szDst:   8,
+			atEOF:   false,
+			in:      "\x80Hello\uFF24\x80",
+			out:     "\ufffdHll",
+			outFull: "\ufffdHll\ufffd",
+			err:     transform.ErrShortDst,
+			errSpan: transform.ErrEndOfSpan,
+			t:       remove,
+		},
+		20: {
+			szDst:   8,
+			atEOF:   false,
+			in:      "Hllll",
+			out:     "Hllll",
+			outFull: "Hllll",
+			t:       remove,
+		}} {
+		tt.check(t, i)
+	}
+}
+
+func TestReplaceIllFormed(t *testing.T) {
+	replace := ReplaceIllFormed()
+
+	for i, tt := range []transformTest{
+		0: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "",
+			out:     "",
+			outFull: "",
+			t:       replace,
+		},
+		1: {
+			szDst:   1,
+			atEOF:   true,
+			in:      "aa",
+			out:     "a",
+			outFull: "aa",
+			err:     transform.ErrShortDst,
+			t:       replace,
+		},
+		2: {
+			szDst:   1,
+			atEOF:   true,
+			in:      "a\x80",
+			out:     "a",
+			outFull: "a\ufffd",
+			err:     transform.ErrShortDst,
+			errSpan: transform.ErrEndOfSpan,
+			t:       replace,
+		},
+		3: {
+			szDst:   1,
+			atEOF:   true,
+			in:      "a\xc2",
+			out:     "a",
+			outFull: "a\ufffd",
+			err:     transform.ErrShortDst,
+			errSpan: transform.ErrEndOfSpan,
+			t:       replace,
+		},
+		4: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "\x80",
+			out:     "\ufffd",
+			outFull: "\ufffd",
+			errSpan: transform.ErrEndOfSpan,
+			t:       replace,
+		},
+		5: {
+			szDst:   large,
+			atEOF:   false,
+			in:      "\x80",
+			out:     "\ufffd",
+			outFull: "\ufffd",
+			errSpan: transform.ErrEndOfSpan,
+			t:       replace,
+		},
+		6: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "\xc2",
+			out:     "\ufffd",
+			outFull: "\ufffd",
+			errSpan: transform.ErrEndOfSpan,
+			t:       replace,
+		},
+		7: {
+			szDst:   large,
+			atEOF:   false,
+			in:      "\xc2",
+			out:     "",
+			outFull: "\ufffd",
+			err:     transform.ErrShortSrc,
+			errSpan: transform.ErrShortSrc,
+			t:       replace,
+		},
+		8: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "Hello world!",
+			out:     "Hello world!",
+			outFull: "Hello world!",
+			t:       replace,
+		},
+		9: {
+			szDst:   large,
+			atEOF:   true,
+			in:      "Hello\x80 w\x80orl\xc2d!\xc2",
+			out:     "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+			outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+			errSpan: transform.ErrEndOfSpan,
+			t:       replace,
+		},
+		10: {
+			szDst:   large,
+			atEOF:   false,
+			in:      "Hello\x80 w\x80orl\xc2d!\xc2",
+			out:     "Hello\ufffd w\ufffdorl\ufffdd!",
+			outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+			err:     transform.ErrShortSrc,
+			errSpan: transform.ErrEndOfSpan,
+			t:       replace,
+		},
+		16: {
+			szDst:   10,
+			atEOF:   false,
+			in:      "\x80Hello\x80",
+			out:     "\ufffdHello",
+			outFull: "\ufffdHello\ufffd",
+			err:     transform.ErrShortDst,
+			errSpan: transform.ErrEndOfSpan,
+			t:       replace,
+		},
+		17: {
+			szDst:   10,
+			atEOF:   false,
+			in:      "\ufffdHello\ufffd",
+			out:     "\ufffdHello",
+			outFull: "\ufffdHello\ufffd",
+			err:     transform.ErrShortDst,
+			t:       replace,
+		},
+	} {
+		tt.check(t, i)
+	}
+}
+
+func TestMapAlloc(t *testing.T) {
+	if n := testtext.AllocsPerRun(3, func() {
+		Map(idem).Transform(nil, nil, false)
+	}); n > 0 {
+		t.Errorf("got %f; want 0", n)
+	}
+}
+
+func rmNop(r rune) bool { return false }
+
+func TestRemoveAlloc(t *testing.T) {
+	if n := testtext.AllocsPerRun(3, func() {
+		Remove(Predicate(rmNop)).Transform(nil, nil, false)
+	}); n > 0 {
+		t.Errorf("got %f; want 0", n)
+	}
+}
+
+func TestReplaceIllFormedAlloc(t *testing.T) {
+	if n := testtext.AllocsPerRun(3, func() {
+		ReplaceIllFormed().Transform(nil, nil, false)
+	}); n > 0 {
+		t.Errorf("got %f; want 0", n)
+	}
+}
+
+func doBench(b *testing.B, t Transformer) {
+	for _, bc := range []struct{ name, data string }{
+		{"ascii", testtext.ASCII},
+		{"3byte", testtext.ThreeByteUTF8},
+	} {
+		dst := make([]byte, 2*len(bc.data))
+		src := []byte(bc.data)
+
+		testtext.Bench(b, bc.name+"/transform", func(b *testing.B) {
+			b.SetBytes(int64(len(src)))
+			for i := 0; i < b.N; i++ {
+				t.Transform(dst, src, true)
+			}
+		})
+		src = t.Bytes(src)
+		t.Reset()
+		testtext.Bench(b, bc.name+"/span", func(b *testing.B) {
+			b.SetBytes(int64(len(src)))
+			for i := 0; i < b.N; i++ {
+				t.Span(src, true)
+			}
+		})
+	}
+}
+
+func BenchmarkRemove(b *testing.B) {
+	doBench(b, Remove(Predicate(func(r rune) bool { return r == 'e' })))
+}
+
+func BenchmarkMapAll(b *testing.B) {
+	doBench(b, Map(func(r rune) rune { return 'a' }))
+}
+
+func BenchmarkMapNone(b *testing.B) {
+	doBench(b, Map(func(r rune) rune { return r }))
+}
+
+func BenchmarkReplaceIllFormed(b *testing.B) {
+	doBench(b, ReplaceIllFormed())
+}
+
+var (
+	input = strings.Repeat("Thé qüick brøwn føx jumps øver the lazy døg. ", 100)
+)