rj_schema 0.2.4 → 0.2.5

data/ext/rj_schema/rapidjson/test/unittest/dtoatest.cpp

@@ -0,0 +1,98 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#include "unittest.h"
+#include "rapidjson/internal/dtoa.h"
+
+#ifdef __GNUC__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(type-limits)
+#endif
+
+using namespace rapidjson::internal;
+
+TEST(dtoa, normal) {
+    char buffer[30];
+
+#define TEST_DTOA(d, a)\
+    *dtoa(d, buffer) = '\0';\
+    EXPECT_STREQ(a, buffer)
+
+    TEST_DTOA(0.0, "0.0");
+    TEST_DTOA(-0.0, "-0.0");
+    TEST_DTOA(1.0, "1.0");
+    TEST_DTOA(-1.0, "-1.0");
+    TEST_DTOA(1.2345, "1.2345");
+    TEST_DTOA(1.2345678, "1.2345678");
+    TEST_DTOA(0.123456789012, "0.123456789012");
+    TEST_DTOA(1234567.8, "1234567.8");
+    TEST_DTOA(-79.39773355813419, "-79.39773355813419");
+    TEST_DTOA(0.000001, "0.000001");
+    TEST_DTOA(0.0000001, "1e-7");
+    TEST_DTOA(1e30, "1e30");
+    TEST_DTOA(1.234567890123456e30, "1.234567890123456e30");
+    TEST_DTOA(5e-324, "5e-324"); // Min subnormal positive double
+    TEST_DTOA(2.225073858507201e-308, "2.225073858507201e-308"); // Max subnormal positive double
+    TEST_DTOA(2.2250738585072014e-308, "2.2250738585072014e-308"); // Min normal positive double
+    TEST_DTOA(1.7976931348623157e308, "1.7976931348623157e308"); // Max double
+
+#undef TEST_DTOA
+}
+
+TEST(dtoa, maxDecimalPlaces) {
+    char buffer[30];
+
+#define TEST_DTOA(m, d, a)\
+    *dtoa(d, buffer, m) = '\0';\
+    EXPECT_STREQ(a, buffer)
+
+    TEST_DTOA(3, 0.0, "0.0");
+    TEST_DTOA(1, 0.0, "0.0");
+    TEST_DTOA(3, -0.0, "-0.0");
+    TEST_DTOA(3, 1.0, "1.0");
+    TEST_DTOA(3, -1.0, "-1.0");
+    TEST_DTOA(3, 1.2345, "1.234");
+    TEST_DTOA(2, 1.2345, "1.23");
+    TEST_DTOA(1, 1.2345, "1.2");
+    TEST_DTOA(3, 1.2345678, "1.234");
+    TEST_DTOA(3, 1.0001, "1.0");
+    TEST_DTOA(2, 1.0001, "1.0");
+    TEST_DTOA(1, 1.0001, "1.0");
+    TEST_DTOA(3, 0.123456789012, "0.123");
+    TEST_DTOA(2, 0.123456789012, "0.12");
+    TEST_DTOA(1, 0.123456789012, "0.1");
+    TEST_DTOA(4, 0.0001, "0.0001");
+    TEST_DTOA(3, 0.0001, "0.0");
+    TEST_DTOA(2, 0.0001, "0.0");
+    TEST_DTOA(1, 0.0001, "0.0");
+    TEST_DTOA(3, 1234567.8, "1234567.8");
+    TEST_DTOA(3, 1e30, "1e30");
+    TEST_DTOA(3, 5e-324, "0.0"); // Min subnormal positive double
+    TEST_DTOA(3, 2.225073858507201e-308, "0.0"); // Max subnormal positive double
+    TEST_DTOA(3, 2.2250738585072014e-308, "0.0"); // Min normal positive double
+    TEST_DTOA(3, 1.7976931348623157e308, "1.7976931348623157e308"); // Max double
+    TEST_DTOA(5, -0.14000000000000001, "-0.14");
+    TEST_DTOA(4, -0.14000000000000001, "-0.14");
+    TEST_DTOA(3, -0.14000000000000001, "-0.14");
+    TEST_DTOA(3, -0.10000000000000001, "-0.1");
+    TEST_DTOA(2, -0.10000000000000001, "-0.1");
+    TEST_DTOA(1, -0.10000000000000001, "-0.1");
+
+#undef TEST_DTOA
+}
+
+
+#ifdef __GNUC__
+RAPIDJSON_DIAG_POP
+#endif

data/ext/rj_schema/rapidjson/test/unittest/encodedstreamtest.cpp

@@ -0,0 +1,313 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#include "unittest.h"
+#include "rapidjson/filereadstream.h"
+#include "rapidjson/filewritestream.h"
+#include "rapidjson/encodedstream.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/memorystream.h"
+#include "rapidjson/memorybuffer.h"
+
+using namespace rapidjson;
+
+class EncodedStreamTest : public ::testing::Test {
+public:
+    EncodedStreamTest() : json_(), length_() {}
+    virtual ~EncodedStreamTest();
+
+    virtual void SetUp() {
+        json_ = ReadFile("utf8.json", true, &length_);
+    }
+
+    virtual void TearDown() {
+        free(json_);
+        json_ = 0;
+    }
+
+private:
+    EncodedStreamTest(const EncodedStreamTest&);
+    EncodedStreamTest& operator=(const EncodedStreamTest&);
+    
+protected:
+    static FILE* Open(const char* filename) {
+        const char *paths[] = {
+            "encodings",
+            "bin/encodings",
+            "../bin/encodings",
+            "../../bin/encodings",
+            "../../../bin/encodings"
+        };
+        char buffer[1024];
+        for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
+            sprintf(buffer, "%s/%s", paths[i], filename);
+            FILE *fp = fopen(buffer, "rb");
+            if (fp)
+                return fp;
+        }
+        return 0;
+    }
+
+    static char *ReadFile(const char* filename, bool appendPath, size_t* outLength) {
+        FILE *fp = appendPath ? Open(filename) : fopen(filename, "rb");
+
+        if (!fp) {
+            *outLength = 0;
+            return 0;
+        }
+
+        fseek(fp, 0, SEEK_END);
+        *outLength = static_cast<size_t>(ftell(fp));
+        fseek(fp, 0, SEEK_SET);
+        char* buffer = static_cast<char*>(malloc(*outLength + 1));
+        size_t readLength = fread(buffer, 1, *outLength, fp);
+        buffer[readLength] = '\0';
+        fclose(fp);
+        return buffer;
+    }
+
+    template <typename FileEncoding, typename MemoryEncoding>
+    void TestEncodedInputStream(const char* filename) {
+        // Test FileReadStream
+        {
+            char buffer[16];
+            FILE *fp = Open(filename);
+            ASSERT_TRUE(fp != 0);
+            FileReadStream fs(fp, buffer, sizeof(buffer));
+            EncodedInputStream<FileEncoding, FileReadStream> eis(fs);
+            StringStream s(json_);
+
+            while (eis.Peek() != '\0') {
+                unsigned expected, actual;
+                EXPECT_TRUE(UTF8<>::Decode(s, &expected));
+                EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual));
+                EXPECT_EQ(expected, actual);
+            }
+            EXPECT_EQ('\0', s.Peek());
+            fclose(fp);
+        }
+
+        // Test MemoryStream
+        {
+            size_t size;
+            char* data = ReadFile(filename, true, &size);
+            MemoryStream ms(data, size);
+            EncodedInputStream<FileEncoding, MemoryStream> eis(ms);
+            StringStream s(json_);
+
+            while (eis.Peek() != '\0') {
+                unsigned expected, actual;
+                EXPECT_TRUE(UTF8<>::Decode(s, &expected));
+                EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual));
+                EXPECT_EQ(expected, actual);
+            }
+            EXPECT_EQ('\0', s.Peek());
+            free(data);
+            EXPECT_EQ(size, eis.Tell());
+        }
+    }
+
+    void TestAutoUTFInputStream(const char *filename, bool expectHasBOM) {
+        // Test FileReadStream
+        {
+            char buffer[16];
+            FILE *fp = Open(filename);
+            ASSERT_TRUE(fp != 0);
+            FileReadStream fs(fp, buffer, sizeof(buffer));
+            AutoUTFInputStream<unsigned, FileReadStream> eis(fs);
+            EXPECT_EQ(expectHasBOM, eis.HasBOM());
+            StringStream s(json_);
+            while (eis.Peek() != '\0') {
+                unsigned expected, actual;
+                EXPECT_TRUE(UTF8<>::Decode(s, &expected));
+                EXPECT_TRUE(AutoUTF<unsigned>::Decode(eis, &actual));
+                EXPECT_EQ(expected, actual);
+            }
+            EXPECT_EQ('\0', s.Peek());
+            fclose(fp);
+        }
+
+        // Test MemoryStream
+        {
+            size_t size;
+            char* data = ReadFile(filename, true, &size);
+            MemoryStream ms(data, size);
+            AutoUTFInputStream<unsigned, MemoryStream> eis(ms);
+            EXPECT_EQ(expectHasBOM, eis.HasBOM());
+            StringStream s(json_);
+
+            while (eis.Peek() != '\0') {
+                unsigned expected, actual;
+                EXPECT_TRUE(UTF8<>::Decode(s, &expected));
+                EXPECT_TRUE(AutoUTF<unsigned>::Decode(eis, &actual));
+                EXPECT_EQ(expected, actual);
+            }
+            EXPECT_EQ('\0', s.Peek());
+            free(data);
+            EXPECT_EQ(size, eis.Tell());
+        }
+    }
+
+    template <typename FileEncoding, typename MemoryEncoding>
+    void TestEncodedOutputStream(const char* expectedFilename, bool putBOM) {
+        // Test FileWriteStream
+        {
+            char filename[L_tmpnam];
+            FILE* fp = TempFile(filename);
+            char buffer[16];
+            FileWriteStream os(fp, buffer, sizeof(buffer));
+            EncodedOutputStream<FileEncoding, FileWriteStream> eos(os, putBOM);
+            StringStream s(json_);
+            while (s.Peek() != '\0') {
+                bool success = Transcoder<UTF8<>, MemoryEncoding>::Transcode(s, eos);
+                EXPECT_TRUE(success);
+            }
+            eos.Flush();
+            fclose(fp);
+            EXPECT_TRUE(CompareFile(filename, expectedFilename));
+            remove(filename);
+        }
+
+        // Test MemoryBuffer
+        {
+            MemoryBuffer mb;
+            EncodedOutputStream<FileEncoding, MemoryBuffer> eos(mb, putBOM);
+            StringStream s(json_);
+            while (s.Peek() != '\0') {
+                bool success = Transcoder<UTF8<>, MemoryEncoding>::Transcode(s, eos);
+                EXPECT_TRUE(success);
+            }
+            eos.Flush();
+            EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename));
+        }
+    }
+
+    void TestAutoUTFOutputStream(UTFType type, bool putBOM, const char *expectedFilename) {
+        // Test FileWriteStream
+        {
+            char filename[L_tmpnam];
+            FILE* fp = TempFile(filename);
+
+            char buffer[16];
+            FileWriteStream os(fp, buffer, sizeof(buffer));
+            AutoUTFOutputStream<unsigned, FileWriteStream> eos(os, type, putBOM);
+            StringStream s(json_);
+            while (s.Peek() != '\0') {
+                bool success = Transcoder<UTF8<>, AutoUTF<unsigned> >::Transcode(s, eos);
+                EXPECT_TRUE(success);
+            }
+            eos.Flush();
+            fclose(fp);
+            EXPECT_TRUE(CompareFile(filename, expectedFilename));
+            remove(filename);
+        }
+
+        // Test MemoryBuffer
+        {
+            MemoryBuffer mb;
+            AutoUTFOutputStream<unsigned, MemoryBuffer> eos(mb, type, putBOM);
+            StringStream s(json_);
+            while (s.Peek() != '\0') {
+                bool success = Transcoder<UTF8<>, AutoUTF<unsigned> >::Transcode(s, eos);
+                EXPECT_TRUE(success);
+            }
+            eos.Flush();
+            EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename));
+        }
+    }
+
+    bool CompareFile(const char* filename, const char* expectedFilename) {
+        size_t actualLength, expectedLength;
+        char* actualBuffer = ReadFile(filename, false, &actualLength);
+        char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength);
+        bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0;
+        free(actualBuffer);
+        free(expectedBuffer);
+        return ret;
+    }
+
+    bool CompareBufferFile(const char* actualBuffer, size_t actualLength, const char* expectedFilename) {
+        size_t expectedLength;
+        char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength);
+        bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0;
+        free(expectedBuffer);
+        return ret;
+    }
+
+    char *json_;
+    size_t length_;
+};
+
+EncodedStreamTest::~EncodedStreamTest() {}
+
+TEST_F(EncodedStreamTest, EncodedInputStream) {
+    TestEncodedInputStream<UTF8<>,    UTF8<>  >("utf8.json");
+    TestEncodedInputStream<UTF8<>,    UTF8<>  >("utf8bom.json");
+    TestEncodedInputStream<UTF16LE<>, UTF16<> >("utf16le.json");
+    TestEncodedInputStream<UTF16LE<>, UTF16<> >("utf16lebom.json");
+    TestEncodedInputStream<UTF16BE<>, UTF16<> >("utf16be.json");
+    TestEncodedInputStream<UTF16BE<>, UTF16<> >("utf16bebom.json");
+    TestEncodedInputStream<UTF32LE<>, UTF32<> >("utf32le.json");
+    TestEncodedInputStream<UTF32LE<>, UTF32<> >("utf32lebom.json");
+    TestEncodedInputStream<UTF32BE<>, UTF32<> >("utf32be.json");
+    TestEncodedInputStream<UTF32BE<>, UTF32<> >("utf32bebom.json");
+}
+
+TEST_F(EncodedStreamTest, AutoUTFInputStream) {
+    TestAutoUTFInputStream("utf8.json",      false);
+    TestAutoUTFInputStream("utf8bom.json",   true);
+    TestAutoUTFInputStream("utf16le.json",   false);
+    TestAutoUTFInputStream("utf16lebom.json",true);
+    TestAutoUTFInputStream("utf16be.json",   false);
+    TestAutoUTFInputStream("utf16bebom.json",true);
+    TestAutoUTFInputStream("utf32le.json",   false);
+    TestAutoUTFInputStream("utf32lebom.json",true);
+    TestAutoUTFInputStream("utf32be.json",   false);
+    TestAutoUTFInputStream("utf32bebom.json", true);
+
+    {
+        // Auto detection fail, use user defined UTF type
+        const char json[] = "{ }";
+        MemoryStream ms(json, sizeof(json));
+        AutoUTFInputStream<unsigned, MemoryStream> eis(ms, kUTF8);
+        EXPECT_FALSE(eis.HasBOM());
+        EXPECT_EQ(kUTF8, eis.GetType());
+    }
+}
+
+TEST_F(EncodedStreamTest, EncodedOutputStream) {
+    TestEncodedOutputStream<UTF8<>,     UTF8<>  >("utf8.json",      false);
+    TestEncodedOutputStream<UTF8<>,     UTF8<>  >("utf8bom.json",   true);
+    TestEncodedOutputStream<UTF16LE<>,  UTF16<> >("utf16le.json",   false);
+    TestEncodedOutputStream<UTF16LE<>,  UTF16<> >("utf16lebom.json",true);
+    TestEncodedOutputStream<UTF16BE<>,  UTF16<> >("utf16be.json",   false);
+    TestEncodedOutputStream<UTF16BE<>,  UTF16<> >("utf16bebom.json",true);
+    TestEncodedOutputStream<UTF32LE<>,  UTF32<> >("utf32le.json",   false);
+    TestEncodedOutputStream<UTF32LE<>,  UTF32<> >("utf32lebom.json",true);
+    TestEncodedOutputStream<UTF32BE<>,  UTF32<> >("utf32be.json",   false);
+    TestEncodedOutputStream<UTF32BE<>,  UTF32<> >("utf32bebom.json",true);
+}
+
+TEST_F(EncodedStreamTest, AutoUTFOutputStream) {
+    TestAutoUTFOutputStream(kUTF8,      false,  "utf8.json");
+    TestAutoUTFOutputStream(kUTF8,      true,   "utf8bom.json");
+    TestAutoUTFOutputStream(kUTF16LE,   false,  "utf16le.json");
+    TestAutoUTFOutputStream(kUTF16LE,   true,   "utf16lebom.json");
+    TestAutoUTFOutputStream(kUTF16BE,   false,  "utf16be.json");
+    TestAutoUTFOutputStream(kUTF16BE,   true,   "utf16bebom.json");
+    TestAutoUTFOutputStream(kUTF32LE,   false,  "utf32le.json");
+    TestAutoUTFOutputStream(kUTF32LE,   true,   "utf32lebom.json");
+    TestAutoUTFOutputStream(kUTF32BE,   false,  "utf32be.json");
+    TestAutoUTFOutputStream(kUTF32BE,   true,   "utf32bebom.json");
+}

data/ext/rj_schema/rapidjson/test/unittest/encodingstest.cpp

@@ -0,0 +1,451 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#include "unittest.h"
+#include "rapidjson/filereadstream.h"
+#include "rapidjson/filewritestream.h"
+#include "rapidjson/encodedstream.h"
+#include "rapidjson/stringbuffer.h"
+
+using namespace rapidjson;
+
+// Verification of encoders/decoders with Hoehrmann's UTF8 decoder
+
+// http://www.unicode.org/Public/UNIDATA/Blocks.txt
+static const unsigned kCodepointRanges[] = {
+    0x0000,     0x007F,     // Basic Latin
+    0x0080,     0x00FF,     // Latin-1 Supplement
+    0x0100,     0x017F,     // Latin Extended-A
+    0x0180,     0x024F,     // Latin Extended-B
+    0x0250,     0x02AF,     // IPA Extensions
+    0x02B0,     0x02FF,     // Spacing Modifier Letters
+    0x0300,     0x036F,     // Combining Diacritical Marks
+    0x0370,     0x03FF,     // Greek and Coptic
+    0x0400,     0x04FF,     // Cyrillic
+    0x0500,     0x052F,     // Cyrillic Supplement
+    0x0530,     0x058F,     // Armenian
+    0x0590,     0x05FF,     // Hebrew
+    0x0600,     0x06FF,     // Arabic
+    0x0700,     0x074F,     // Syriac
+    0x0750,     0x077F,     // Arabic Supplement
+    0x0780,     0x07BF,     // Thaana
+    0x07C0,     0x07FF,     // NKo
+    0x0800,     0x083F,     // Samaritan
+    0x0840,     0x085F,     // Mandaic
+    0x0900,     0x097F,     // Devanagari
+    0x0980,     0x09FF,     // Bengali
+    0x0A00,     0x0A7F,     // Gurmukhi
+    0x0A80,     0x0AFF,     // Gujarati
+    0x0B00,     0x0B7F,     // Oriya
+    0x0B80,     0x0BFF,     // Tamil
+    0x0C00,     0x0C7F,     // Telugu
+    0x0C80,     0x0CFF,     // Kannada
+    0x0D00,     0x0D7F,     // Malayalam
+    0x0D80,     0x0DFF,     // Sinhala
+    0x0E00,     0x0E7F,     // Thai
+    0x0E80,     0x0EFF,     // Lao
+    0x0F00,     0x0FFF,     // Tibetan
+    0x1000,     0x109F,     // Myanmar
+    0x10A0,     0x10FF,     // Georgian
+    0x1100,     0x11FF,     // Hangul Jamo
+    0x1200,     0x137F,     // Ethiopic
+    0x1380,     0x139F,     // Ethiopic Supplement
+    0x13A0,     0x13FF,     // Cherokee
+    0x1400,     0x167F,     // Unified Canadian Aboriginal Syllabics
+    0x1680,     0x169F,     // Ogham
+    0x16A0,     0x16FF,     // Runic
+    0x1700,     0x171F,     // Tagalog
+    0x1720,     0x173F,     // Hanunoo
+    0x1740,     0x175F,     // Buhid
+    0x1760,     0x177F,     // Tagbanwa
+    0x1780,     0x17FF,     // Khmer
+    0x1800,     0x18AF,     // Mongolian
+    0x18B0,     0x18FF,     // Unified Canadian Aboriginal Syllabics Extended
+    0x1900,     0x194F,     // Limbu
+    0x1950,     0x197F,     // Tai Le
+    0x1980,     0x19DF,     // New Tai Lue
+    0x19E0,     0x19FF,     // Khmer Symbols
+    0x1A00,     0x1A1F,     // Buginese
+    0x1A20,     0x1AAF,     // Tai Tham
+    0x1B00,     0x1B7F,     // Balinese
+    0x1B80,     0x1BBF,     // Sundanese
+    0x1BC0,     0x1BFF,     // Batak
+    0x1C00,     0x1C4F,     // Lepcha
+    0x1C50,     0x1C7F,     // Ol Chiki
+    0x1CD0,     0x1CFF,     // Vedic Extensions
+    0x1D00,     0x1D7F,     // Phonetic Extensions
+    0x1D80,     0x1DBF,     // Phonetic Extensions Supplement
+    0x1DC0,     0x1DFF,     // Combining Diacritical Marks Supplement
+    0x1E00,     0x1EFF,     // Latin Extended Additional
+    0x1F00,     0x1FFF,     // Greek Extended
+    0x2000,     0x206F,     // General Punctuation
+    0x2070,     0x209F,     // Superscripts and Subscripts
+    0x20A0,     0x20CF,     // Currency Symbols
+    0x20D0,     0x20FF,     // Combining Diacritical Marks for Symbols
+    0x2100,     0x214F,     // Letterlike Symbols
+    0x2150,     0x218F,     // Number Forms
+    0x2190,     0x21FF,     // Arrows
+    0x2200,     0x22FF,     // Mathematical Operators
+    0x2300,     0x23FF,     // Miscellaneous Technical
+    0x2400,     0x243F,     // Control Pictures
+    0x2440,     0x245F,     // Optical Character Recognition
+    0x2460,     0x24FF,     // Enclosed Alphanumerics
+    0x2500,     0x257F,     // Box Drawing
+    0x2580,     0x259F,     // Block Elements
+    0x25A0,     0x25FF,     // Geometric Shapes
+    0x2600,     0x26FF,     // Miscellaneous Symbols
+    0x2700,     0x27BF,     // Dingbats
+    0x27C0,     0x27EF,     // Miscellaneous Mathematical Symbols-A
+    0x27F0,     0x27FF,     // Supplemental Arrows-A
+    0x2800,     0x28FF,     // Braille Patterns
+    0x2900,     0x297F,     // Supplemental Arrows-B
+    0x2980,     0x29FF,     // Miscellaneous Mathematical Symbols-B
+    0x2A00,     0x2AFF,     // Supplemental Mathematical Operators
+    0x2B00,     0x2BFF,     // Miscellaneous Symbols and Arrows
+    0x2C00,     0x2C5F,     // Glagolitic
+    0x2C60,     0x2C7F,     // Latin Extended-C
+    0x2C80,     0x2CFF,     // Coptic
+    0x2D00,     0x2D2F,     // Georgian Supplement
+    0x2D30,     0x2D7F,     // Tifinagh
+    0x2D80,     0x2DDF,     // Ethiopic Extended
+    0x2DE0,     0x2DFF,     // Cyrillic Extended-A
+    0x2E00,     0x2E7F,     // Supplemental Punctuation
+    0x2E80,     0x2EFF,     // CJK Radicals Supplement
+    0x2F00,     0x2FDF,     // Kangxi Radicals
+    0x2FF0,     0x2FFF,     // Ideographic Description Characters
+    0x3000,     0x303F,     // CJK Symbols and Punctuation
+    0x3040,     0x309F,     // Hiragana
+    0x30A0,     0x30FF,     // Katakana
+    0x3100,     0x312F,     // Bopomofo
+    0x3130,     0x318F,     // Hangul Compatibility Jamo
+    0x3190,     0x319F,     // Kanbun
+    0x31A0,     0x31BF,     // Bopomofo Extended
+    0x31C0,     0x31EF,     // CJK Strokes
+    0x31F0,     0x31FF,     // Katakana Phonetic Extensions
+    0x3200,     0x32FF,     // Enclosed CJK Letters and Months
+    0x3300,     0x33FF,     // CJK Compatibility
+    0x3400,     0x4DBF,     // CJK Unified Ideographs Extension A
+    0x4DC0,     0x4DFF,     // Yijing Hexagram Symbols
+    0x4E00,     0x9FFF,     // CJK Unified Ideographs
+    0xA000,     0xA48F,     // Yi Syllables
+    0xA490,     0xA4CF,     // Yi Radicals
+    0xA4D0,     0xA4FF,     // Lisu
+    0xA500,     0xA63F,     // Vai
+    0xA640,     0xA69F,     // Cyrillic Extended-B
+    0xA6A0,     0xA6FF,     // Bamum
+    0xA700,     0xA71F,     // Modifier Tone Letters
+    0xA720,     0xA7FF,     // Latin Extended-D
+    0xA800,     0xA82F,     // Syloti Nagri
+    0xA830,     0xA83F,     // Common Indic Number Forms
+    0xA840,     0xA87F,     // Phags-pa
+    0xA880,     0xA8DF,     // Saurashtra
+    0xA8E0,     0xA8FF,     // Devanagari Extended
+    0xA900,     0xA92F,     // Kayah Li
+    0xA930,     0xA95F,     // Rejang
+    0xA960,     0xA97F,     // Hangul Jamo Extended-A
+    0xA980,     0xA9DF,     // Javanese
+    0xAA00,     0xAA5F,     // Cham
+    0xAA60,     0xAA7F,     // Myanmar Extended-A
+    0xAA80,     0xAADF,     // Tai Viet
+    0xAB00,     0xAB2F,     // Ethiopic Extended-A
+    0xABC0,     0xABFF,     // Meetei Mayek
+    0xAC00,     0xD7AF,     // Hangul Syllables
+    0xD7B0,     0xD7FF,     // Hangul Jamo Extended-B
+    //0xD800,       0xDB7F,     // High Surrogates
+    //0xDB80,       0xDBFF,     // High Private Use Surrogates
+    //0xDC00,       0xDFFF,     // Low Surrogates
+    0xE000,     0xF8FF,     // Private Use Area
+    0xF900,     0xFAFF,     // CJK Compatibility Ideographs
+    0xFB00,     0xFB4F,     // Alphabetic Presentation Forms
+    0xFB50,     0xFDFF,     // Arabic Presentation Forms-A
+    0xFE00,     0xFE0F,     // Variation Selectors
+    0xFE10,     0xFE1F,     // Vertical Forms
+    0xFE20,     0xFE2F,     // Combining Half Marks
+    0xFE30,     0xFE4F,     // CJK Compatibility Forms
+    0xFE50,     0xFE6F,     // Small Form Variants
+    0xFE70,     0xFEFF,     // Arabic Presentation Forms-B
+    0xFF00,     0xFFEF,     // Halfwidth and Fullwidth Forms
+    0xFFF0,     0xFFFF,     // Specials
+    0x10000,    0x1007F,    // Linear B Syllabary
+    0x10080,    0x100FF,    // Linear B Ideograms
+    0x10100,    0x1013F,    // Aegean Numbers
+    0x10140,    0x1018F,    // Ancient Greek Numbers
+    0x10190,    0x101CF,    // Ancient Symbols
+    0x101D0,    0x101FF,    // Phaistos Disc
+    0x10280,    0x1029F,    // Lycian
+    0x102A0,    0x102DF,    // Carian
+    0x10300,    0x1032F,    // Old Italic
+    0x10330,    0x1034F,    // Gothic
+    0x10380,    0x1039F,    // Ugaritic
+    0x103A0,    0x103DF,    // Old Persian
+    0x10400,    0x1044F,    // Deseret
+    0x10450,    0x1047F,    // Shavian
+    0x10480,    0x104AF,    // Osmanya
+    0x10800,    0x1083F,    // Cypriot Syllabary
+    0x10840,    0x1085F,    // Imperial Aramaic
+    0x10900,    0x1091F,    // Phoenician
+    0x10920,    0x1093F,    // Lydian
+    0x10A00,    0x10A5F,    // Kharoshthi
+    0x10A60,    0x10A7F,    // Old South Arabian
+    0x10B00,    0x10B3F,    // Avestan
+    0x10B40,    0x10B5F,    // Inscriptional Parthian
+    0x10B60,    0x10B7F,    // Inscriptional Pahlavi
+    0x10C00,    0x10C4F,    // Old Turkic
+    0x10E60,    0x10E7F,    // Rumi Numeral Symbols
+    0x11000,    0x1107F,    // Brahmi
+    0x11080,    0x110CF,    // Kaithi
+    0x12000,    0x123FF,    // Cuneiform
+    0x12400,    0x1247F,    // Cuneiform Numbers and Punctuation
+    0x13000,    0x1342F,    // Egyptian Hieroglyphs
+    0x16800,    0x16A3F,    // Bamum Supplement
+    0x1B000,    0x1B0FF,    // Kana Supplement
+    0x1D000,    0x1D0FF,    // Byzantine Musical Symbols
+    0x1D100,    0x1D1FF,    // Musical Symbols
+    0x1D200,    0x1D24F,    // Ancient Greek Musical Notation
+    0x1D300,    0x1D35F,    // Tai Xuan Jing Symbols
+    0x1D360,    0x1D37F,    // Counting Rod Numerals
+    0x1D400,    0x1D7FF,    // Mathematical Alphanumeric Symbols
+    0x1F000,    0x1F02F,    // Mahjong Tiles
+    0x1F030,    0x1F09F,    // Domino Tiles
+    0x1F0A0,    0x1F0FF,    // Playing Cards
+    0x1F100,    0x1F1FF,    // Enclosed Alphanumeric Supplement
+    0x1F200,    0x1F2FF,    // Enclosed Ideographic Supplement
+    0x1F300,    0x1F5FF,    // Miscellaneous Symbols And Pictographs
+    0x1F600,    0x1F64F,    // Emoticons
+    0x1F680,    0x1F6FF,    // Transport And Map Symbols
+    0x1F700,    0x1F77F,    // Alchemical Symbols
+    0x20000,    0x2A6DF,    // CJK Unified Ideographs Extension B
+    0x2A700,    0x2B73F,    // CJK Unified Ideographs Extension C
+    0x2B740,    0x2B81F,    // CJK Unified Ideographs Extension D
+    0x2F800,    0x2FA1F,    // CJK Compatibility Ideographs Supplement
+    0xE0000,    0xE007F,    // Tags
+    0xE0100,    0xE01EF,    // Variation Selectors Supplement
+    0xF0000,    0xFFFFF,    // Supplementary Private Use Area-A
+    0x100000,   0x10FFFF,   // Supplementary Private Use Area-B
+    0xFFFFFFFF
+};
+
+// Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
+
+#define UTF8_ACCEPT 0u
+
+static const unsigned char utf8d[] = {
+    // The first part of the table maps bytes to character classes that
+    // to reduce the size of the transition table and create bitmasks.
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+    7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+    8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+    10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
+
+    // The second part is a transition table that maps a combination
+    // of a state of the automaton and a character class to a state.
+    0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
+    12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
+    12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
+    12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
+    12,36,12,12,12,12,12,12,12,12,12,12, 
+};
+
+static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) {
+    unsigned type = utf8d[byte];
+
+    *codep = (*state != UTF8_ACCEPT) ?
+        (byte & 0x3fu) | (*codep << 6) :
+    (0xffu >> type) & (byte);
+
+    *state = utf8d[256 + *state + type];
+    return *state;
+}
+
+//static bool IsUTF8(unsigned char* s) {
+//  unsigned codepoint, state = 0;
+//
+//  while (*s)
+//      decode(&state, &codepoint, *s++);
+//
+//  return state == UTF8_ACCEPT;
+//}
+
+TEST(EncodingsTest, UTF8) {
+    StringBuffer os, os2;
+    for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
+        for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
+            os.Clear();
+            UTF8<>::Encode(os, codepoint);
+            const char* encodedStr = os.GetString();
+
+            // Decode with Hoehrmann
+            {
+                unsigned decodedCodepoint = 0;
+                unsigned state = 0;
+
+                unsigned decodedCount = 0;
+                for (const char* s = encodedStr; *s; ++s)
+                    if (!decode(&state, &decodedCodepoint, static_cast<unsigned char>(*s))) {
+                        EXPECT_EQ(codepoint, decodedCodepoint);
+                        decodedCount++;
+                    }
+
+                if (*encodedStr) {                  // This decoder cannot handle U+0000
+                    EXPECT_EQ(1u, decodedCount);    // Should only contain one code point
+                }
+
+                EXPECT_EQ(UTF8_ACCEPT, state);
+                if (UTF8_ACCEPT != state)
+                    std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+            }
+
+            // Decode
+            {
+                StringStream is(encodedStr);
+                unsigned decodedCodepoint;
+                bool result = UTF8<>::Decode(is, &decodedCodepoint);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(codepoint, decodedCodepoint);
+                if (!result || codepoint != decodedCodepoint)
+                    std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+            }
+
+            // Validate
+            {
+                StringStream is(encodedStr);
+                os2.Clear();
+                bool result = UTF8<>::Validate(is, os2);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
+            }
+        }
+    }
+}
+
+TEST(EncodingsTest, UTF16) {
+    GenericStringBuffer<UTF16<> > os, os2;
+    GenericStringBuffer<UTF8<> > utf8os;
+    for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
+        for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
+            os.Clear();
+            UTF16<>::Encode(os, codepoint);
+            const UTF16<>::Ch* encodedStr = os.GetString();
+
+            // Encode with Hoehrmann's code
+            if (codepoint != 0) // cannot handle U+0000
+            {
+                // encode with UTF8<> first
+                utf8os.Clear();
+                UTF8<>::Encode(utf8os, codepoint);
+
+                // transcode from UTF8 to UTF16 with Hoehrmann's code
+                unsigned decodedCodepoint = 0;
+                unsigned state = 0;
+                UTF16<>::Ch buffer[3], *p = &buffer[0];
+                for (const char* s = utf8os.GetString(); *s; ++s) {
+                    if (!decode(&state, &decodedCodepoint, static_cast<unsigned char>(*s)))
+                        break;
+                }
+
+                if (codepoint <= 0xFFFF)
+                    *p++ = static_cast<UTF16<>::Ch>(decodedCodepoint);
+                else {
+                    // Encode code points above U+FFFF as surrogate pair.
+                    *p++ = static_cast<UTF16<>::Ch>(0xD7C0 + (decodedCodepoint >> 10));
+                    *p++ = static_cast<UTF16<>::Ch>(0xDC00 + (decodedCodepoint & 0x3FF));
+                }
+                *p++ = '\0';
+
+                EXPECT_EQ(0, StrCmp(buffer, encodedStr));
+            }
+
+            // Decode
+            {
+                GenericStringStream<UTF16<> > is(encodedStr);
+                unsigned decodedCodepoint;
+                bool result = UTF16<>::Decode(is, &decodedCodepoint);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(codepoint, decodedCodepoint);         
+                if (!result || codepoint != decodedCodepoint)
+                    std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+            }
+
+            // Validate
+            {
+                GenericStringStream<UTF16<> > is(encodedStr);
+                os2.Clear();
+                bool result = UTF16<>::Validate(is, os2);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
+            }
+        }
+    }
+}
+
+TEST(EncodingsTest, UTF32) {
+    GenericStringBuffer<UTF32<> > os, os2;
+    for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
+        for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
+            os.Clear();
+            UTF32<>::Encode(os, codepoint);
+            const UTF32<>::Ch* encodedStr = os.GetString();
+
+            // Decode
+            {
+                GenericStringStream<UTF32<> > is(encodedStr);
+                unsigned decodedCodepoint;
+                bool result = UTF32<>::Decode(is, &decodedCodepoint);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(codepoint, decodedCodepoint);         
+                if (!result || codepoint != decodedCodepoint)
+                    std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+            }
+
+            // Validate
+            {
+                GenericStringStream<UTF32<> > is(encodedStr);
+                os2.Clear();
+                bool result = UTF32<>::Validate(is, os2);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
+            }
+        }
+    }
+}
+
+TEST(EncodingsTest, ASCII) {
+    StringBuffer os, os2;
+    for (unsigned codepoint = 0; codepoint < 128; codepoint++) {
+        os.Clear();
+        ASCII<>::Encode(os, codepoint);
+        const ASCII<>::Ch* encodedStr = os.GetString();
+        {
+            StringStream is(encodedStr);
+            unsigned decodedCodepoint;
+            bool result = ASCII<>::Decode(is, &decodedCodepoint);
+            if (!result || codepoint != decodedCodepoint)
+                std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+        }
+
+        // Validate
+        {
+            StringStream is(encodedStr);
+            os2.Clear();
+            bool result = ASCII<>::Validate(is, os2);
+            EXPECT_TRUE(result);
+            EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
+        }
+    }
+}