alphatau 0.1.0

data/ext/alphatau/native/sdl/test/testyuv_cvt.c

@@ -0,0 +1,300 @@
+/*
+  Copyright (C) 1997-2018 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely.
+*/
+
+#include "SDL.h"
+
+#include "testyuv_cvt.h"
+
+
+static float clip3(float x, float y, float z)
+{
+    return ((z < x) ? x : ((z > y) ? y : z));
+}
+
+static void RGBtoYUV(Uint8 * rgb, int *yuv, SDL_YUV_CONVERSION_MODE mode, int monochrome, int luminance)
+{
+    if (mode == SDL_YUV_CONVERSION_JPEG) {
+        /* Full range YUV */
+        yuv[0] = (int)(0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2]);
+        yuv[1] = (int)((rgb[2] - yuv[0]) * 0.565 + 128);
+        yuv[2] = (int)((rgb[0] - yuv[0]) * 0.713 + 128);
+    } else {
+        // This formula is from Microsoft's documentation:
+        // https://msdn.microsoft.com/en-us/library/windows/desktop/dd206750(v=vs.85).aspx
+        // L = Kr * R + Kb * B + (1 - Kr - Kb) * G
+        // Y =                   floor(2^(M-8) * (219*(L-Z)/S + 16) + 0.5);
+        // U = clip3(0, (2^M)-1, floor(2^(M-8) * (112*(B-L) / ((1-Kb)*S) + 128) + 0.5));
+        // V = clip3(0, (2^M)-1, floor(2^(M-8) * (112*(R-L) / ((1-Kr)*S) + 128) + 0.5));
+        float S, Z, R, G, B, L, Kr, Kb, Y, U, V;
+
+        if (mode == SDL_YUV_CONVERSION_BT709) {
+            /* BT.709 */
+            Kr = 0.2126f;
+            Kb = 0.0722f;
+        } else {
+            /* BT.601 */
+            Kr = 0.299f;
+            Kb = 0.114f;
+        }
+
+        S = 255.0f;
+        Z = 0.0f;
+        R = rgb[0];
+        G = rgb[1];
+        B = rgb[2];
+        L = Kr * R + Kb * B + (1 - Kr - Kb) * G;
+        Y = (Uint8)SDL_floorf((219*(L-Z)/S + 16) + 0.5f);
+        U = (Uint8)clip3(0, 255, SDL_floorf((112.0f*(B-L) / ((1.0f-Kb)*S) + 128) + 0.5f));
+        V = (Uint8)clip3(0, 255, SDL_floorf((112.0f*(R-L) / ((1.0f-Kr)*S) + 128) + 0.5f));
+
+        yuv[0] = (Uint8)Y;
+        yuv[1] = (Uint8)U;
+        yuv[2] = (Uint8)V;
+    }
+
+    if (monochrome) {
+        yuv[1] = 128;
+        yuv[2] = 128;
+    }
+
+    if (luminance != 100) {
+        yuv[0] = yuv[0] * luminance / 100;
+        if (yuv[0] > 255)
+            yuv[0] = 255;
+    }
+}
+
+static void ConvertRGBtoPlanar2x2(Uint32 format, Uint8 *src, int pitch, Uint8 *out, int w, int h, SDL_YUV_CONVERSION_MODE mode, int monochrome, int luminance)
+{
+    int x, y;
+    int yuv[4][3];
+    Uint8 *Y1, *Y2, *U, *V;
+    Uint8 *rgb1, *rgb2;
+    int rgb_row_advance = (pitch - w*3) + pitch;
+    int UV_advance;
+
+    rgb1 = src;
+    rgb2 = src + pitch;
+
+    Y1 = out;
+    Y2 = Y1 + w;
+    switch (format) {
+    case SDL_PIXELFORMAT_YV12:
+        V = (Y1 + h * w);
+        U = V + ((h + 1)/2)*((w + 1)/2);
+        UV_advance = 1;
+        break;
+    case SDL_PIXELFORMAT_IYUV:
+        U = (Y1 + h * w);
+        V = U + ((h + 1)/2)*((w + 1)/2);
+        UV_advance = 1;
+        break;
+    case SDL_PIXELFORMAT_NV12:
+        U = (Y1 + h * w);
+        V = U + 1;
+        UV_advance = 2;
+        break;
+    case SDL_PIXELFORMAT_NV21:
+        V = (Y1 + h * w);
+        U = V + 1;
+        UV_advance = 2;
+        break;
+    default:
+        SDL_assert(!"Unsupported planar YUV format");
+        return;
+    }
+
+    for (y = 0; y < (h - 1); y += 2) {
+        for (x = 0; x < (w - 1); x += 2) {
+            RGBtoYUV(rgb1, yuv[0], mode, monochrome, luminance);
+            rgb1 += 3;
+            *Y1++ = (Uint8)yuv[0][0];
+
+            RGBtoYUV(rgb1, yuv[1], mode, monochrome, luminance);
+            rgb1 += 3;
+            *Y1++ = (Uint8)yuv[1][0];
+
+            RGBtoYUV(rgb2, yuv[2], mode, monochrome, luminance);
+            rgb2 += 3;
+            *Y2++ = (Uint8)yuv[2][0];
+
+            RGBtoYUV(rgb2, yuv[3], mode, monochrome, luminance);
+            rgb2 += 3;
+            *Y2++ = (Uint8)yuv[3][0];
+
+            *U = (Uint8)SDL_floorf((yuv[0][1] + yuv[1][1] + yuv[2][1] + yuv[3][1])/4.0f + 0.5f);
+            U += UV_advance;
+
+            *V = (Uint8)SDL_floorf((yuv[0][2] + yuv[1][2] + yuv[2][2] + yuv[3][2])/4.0f + 0.5f);
+            V += UV_advance;
+        }
+        /* Last column */
+        if (x == (w - 1)) {
+            RGBtoYUV(rgb1, yuv[0], mode, monochrome, luminance);
+            rgb1 += 3;
+            *Y1++ = (Uint8)yuv[0][0];
+
+            RGBtoYUV(rgb2, yuv[2], mode, monochrome, luminance);
+            rgb2 += 3;
+            *Y2++ = (Uint8)yuv[2][0];
+
+            *U = (Uint8)SDL_floorf((yuv[0][1] + yuv[2][1])/2.0f + 0.5f);
+            U += UV_advance;
+
+            *V = (Uint8)SDL_floorf((yuv[0][2] + yuv[2][2])/2.0f + 0.5f);
+            V += UV_advance;
+        }
+        Y1 += w;
+        Y2 += w;
+        rgb1 += rgb_row_advance;
+        rgb2 += rgb_row_advance;
+    }
+    /* Last row */
+    if (y == (h - 1)) {
+        for (x = 0; x < (w - 1); x += 2) {
+            RGBtoYUV(rgb1, yuv[0], mode, monochrome, luminance);
+            rgb1 += 3;
+            *Y1++ = (Uint8)yuv[0][0];
+
+            RGBtoYUV(rgb1, yuv[1], mode, monochrome, luminance);
+            rgb1 += 3;
+            *Y1++ = (Uint8)yuv[1][0];
+
+            *U = (Uint8)SDL_floorf((yuv[0][1] + yuv[1][1])/2.0f + 0.5f);
+            U += UV_advance;
+
+            *V = (Uint8)SDL_floorf((yuv[0][2] + yuv[1][2])/2.0f + 0.5f);
+            V += UV_advance;
+        }
+        /* Last column */
+        if (x == (w - 1)) {
+            RGBtoYUV(rgb1, yuv[0], mode, monochrome, luminance);
+            *Y1++ = (Uint8)yuv[0][0];
+
+            *U = (Uint8)yuv[0][1];
+            U += UV_advance;
+
+            *V = (Uint8)yuv[0][2];
+            V += UV_advance;
+        }
+    }
+}
+
+static void ConvertRGBtoPacked4(Uint32 format, Uint8 *src, int pitch, Uint8 *out, int w, int h, SDL_YUV_CONVERSION_MODE mode, int monochrome, int luminance)
+{
+    int x, y;
+    int yuv[2][3];
+    Uint8 *Y1, *Y2, *U, *V;
+    Uint8 *rgb;
+    int rgb_row_advance = (pitch - w*3);
+
+    rgb = src;
+
+    switch (format) {
+    case SDL_PIXELFORMAT_YUY2:
+        Y1 = out;
+        U = out+1;
+        Y2 = out+2;
+        V = out+3;
+        break;
+    case SDL_PIXELFORMAT_UYVY:
+        U = out;
+        Y1 = out+1;
+        V = out+2;
+        Y2 = out+3;
+        break;
+    case SDL_PIXELFORMAT_YVYU:
+        Y1 = out;
+        V = out+1;
+        Y2 = out+2;
+        U = out+3;
+        break;
+    default:
+        SDL_assert(!"Unsupported packed YUV format");
+        return;
+    }
+
+    for (y = 0; y < h; ++y) {
+        for (x = 0; x < (w - 1); x += 2) {
+            RGBtoYUV(rgb, yuv[0], mode, monochrome, luminance);
+            rgb += 3;
+            *Y1 = (Uint8)yuv[0][0];
+            Y1 += 4;
+
+            RGBtoYUV(rgb, yuv[1], mode, monochrome, luminance);
+            rgb += 3;
+            *Y2 = (Uint8)yuv[1][0];
+            Y2 += 4;
+
+            *U = (Uint8)SDL_floorf((yuv[0][1] + yuv[1][1])/2.0f + 0.5f);
+            U += 4;
+
+            *V = (Uint8)SDL_floorf((yuv[0][2] + yuv[1][2])/2.0f + 0.5f);
+            V += 4;
+        }
+        /* Last column */
+        if (x == (w - 1)) {
+            RGBtoYUV(rgb, yuv[0], mode, monochrome, luminance);
+            rgb += 3;
+            *Y2 = *Y1 = (Uint8)yuv[0][0];
+            Y1 += 4;
+            Y2 += 4;
+
+            *U = (Uint8)yuv[0][1];
+            U += 4;
+
+            *V = (Uint8)yuv[0][2];
+            V += 4;
+        }
+        rgb += rgb_row_advance;
+    }
+}
+
+SDL_bool ConvertRGBtoYUV(Uint32 format, Uint8 *src, int pitch, Uint8 *out, int w, int h, SDL_YUV_CONVERSION_MODE mode, int monochrome, int luminance)
+{
+    switch (format)
+    {
+    case SDL_PIXELFORMAT_YV12:
+    case SDL_PIXELFORMAT_IYUV:
+    case SDL_PIXELFORMAT_NV12:
+    case SDL_PIXELFORMAT_NV21:
+        ConvertRGBtoPlanar2x2(format, src, pitch, out, w, h, mode, monochrome, luminance);
+        return SDL_TRUE;
+    case SDL_PIXELFORMAT_YUY2:
+    case SDL_PIXELFORMAT_UYVY:
+    case SDL_PIXELFORMAT_YVYU:
+        ConvertRGBtoPacked4(format, src, pitch, out, w, h, mode, monochrome, luminance);
+        return SDL_TRUE;
+    default:
+        return SDL_FALSE;
+    }
+}
+
+int CalculateYUVPitch(Uint32 format, int width)
+{
+    switch (format)
+    {
+    case SDL_PIXELFORMAT_YV12:
+    case SDL_PIXELFORMAT_IYUV:
+    case SDL_PIXELFORMAT_NV12:
+    case SDL_PIXELFORMAT_NV21:
+        return width;
+    case SDL_PIXELFORMAT_YUY2:
+    case SDL_PIXELFORMAT_UYVY:
+    case SDL_PIXELFORMAT_YVYU:
+        return 4*((width + 1)/2);
+    default:
+        return 0;
+    }
+}
+
+/* vi: set ts=4 sw=4 expandtab: */

data/ext/alphatau/native/sdl/test/testyuv_cvt.h

@@ -0,0 +1,16 @@
+/*
+  Copyright (C) 1997-2018 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely.
+*/
+
+/* These functions are designed for testing correctness, not for speed */
+
+extern SDL_bool ConvertRGBtoYUV(Uint32 format, Uint8 *src, int pitch, Uint8 *out, int w, int h, SDL_YUV_CONVERSION_MODE mode, int monochrome, int luminance);
+extern int CalculateYUVPitch(Uint32 format, int width);

data/ext/alphatau/native/sdl/test/torturethread.c

@@ -0,0 +1,113 @@
+/*
+  Copyright (C) 1997-2018 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely.
+*/
+
+/* Simple test of the SDL threading code */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <string.h>
+
+#include "SDL.h"
+
+#define NUMTHREADS 10
+
+static SDL_atomic_t time_for_threads_to_die[NUMTHREADS];
+
+/* Call this instead of exit(), so we can clean up SDL: atexit() is evil. */
+static void
+quit(int rc)
+{
+    SDL_Quit();
+    exit(rc);
+}
+
+int SDLCALL
+SubThreadFunc(void *data)
+{
+    while (!*(int volatile *) data) {
+        ;                       /* SDL_Delay(10); *//* do nothing */
+    }
+    return 0;
+}
+
+int SDLCALL
+ThreadFunc(void *data)
+{
+    SDL_Thread *sub_threads[NUMTHREADS];
+    int flags[NUMTHREADS];
+    int i;
+    int tid = (int) (uintptr_t) data;
+
+    SDL_Log("Creating Thread %d\n", tid);
+
+    for (i = 0; i < NUMTHREADS; i++) {
+        char name[64];
+        SDL_snprintf(name, sizeof (name), "Child%d_%d", tid, i);
+        flags[i] = 0;
+        sub_threads[i] = SDL_CreateThread(SubThreadFunc, name, &flags[i]);
+    }
+
+    SDL_Log("Thread '%d' waiting for signal\n", tid);
+    while (SDL_AtomicGet(&time_for_threads_to_die[tid]) != 1) {
+        ;                       /* do nothing */
+    }
+
+    SDL_Log("Thread '%d' sending signals to subthreads\n", tid);
+    for (i = 0; i < NUMTHREADS; i++) {
+        flags[i] = 1;
+        SDL_WaitThread(sub_threads[i], NULL);
+    }
+
+    SDL_Log("Thread '%d' exiting!\n", tid);
+
+    return 0;
+}
+
+int
+main(int argc, char *argv[])
+{
+    SDL_Thread *threads[NUMTHREADS];
+    int i;
+
+    /* Enable standard application logging */
+    SDL_LogSetPriority(SDL_LOG_CATEGORY_APPLICATION, SDL_LOG_PRIORITY_INFO);
+
+    /* Load the SDL library */
+    if (SDL_Init(0) < 0) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't initialize SDL: %s\n", SDL_GetError());
+        return (1);
+    }
+
+    signal(SIGSEGV, SIG_DFL);
+    for (i = 0; i < NUMTHREADS; i++) {
+        char name[64];
+        SDL_snprintf(name, sizeof (name), "Parent%d", i);
+        SDL_AtomicSet(&time_for_threads_to_die[i], 0);
+        threads[i] = SDL_CreateThread(ThreadFunc, name, (void*) (uintptr_t) i);
+
+        if (threads[i] == NULL) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create thread: %s\n", SDL_GetError());
+            quit(1);
+        }
+    }
+
+    for (i = 0; i < NUMTHREADS; i++) {
+        SDL_AtomicSet(&time_for_threads_to_die[i], 1);
+    }
+
+    for (i = 0; i < NUMTHREADS; i++) {
+        SDL_WaitThread(threads[i], NULL);
+    }
+    SDL_Quit();
+    return (0);
+}

data/ext/alphatau/native/sdl/test/utf8.txt

@@ -0,0 +1,287 @@
+UTF-8 decoder capability and stress test
+----------------------------------------
+
+Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> - 2003-02-19
+
+This test file can help you examine, how your UTF-8 decoder handles
+various types of correct, malformed, or otherwise interesting UTF-8
+sequences. This file is not meant to be a conformance test. It does
+not prescribes any particular outcome and therefore there is no way to
+"pass" or "fail" this test file, even though the texts suggests a
+preferable decoder behaviour at some places. The aim is instead to
+help you think about and test the behaviour of your UTF-8 on a
+systematic collection of unusual inputs. Experience so far suggests
+that most first-time authors of UTF-8 decoders find at least one
+serious problem in their decoder by using this file.
+
+The test lines below cover boundary conditions, malformed UTF-8
+sequences as well as correctly encoded UTF-8 sequences of Unicode code
+points that should never occur in a correct UTF-8 file.
+
+According to ISO 10646-1:2000, sections D.7 and 2.3c, a device
+receiving UTF-8 shall interpret a "malformed sequence in the same way
+that it interprets a character that is outside the adopted subset" and
+"characters that are not within the adopted subset shall be indicated
+to the user" by a receiving device. A quite commonly used approach in
+UTF-8 decoders is to replace any malformed UTF-8 sequence by a
+replacement character (U+FFFD), which looks a bit like an inverted
+question mark, or a similar symbol. It might be a good idea to
+visually distinguish a malformed UTF-8 sequence from a correctly
+encoded Unicode character that is just not available in the current
+font but otherwise fully legal, even though ISO 10646-1 doesn't
+mandate this. In any case, just ignoring malformed sequences or
+unavailable characters does not conform to ISO 10646, will make
+debugging more difficult, and can lead to user confusion.
+
+Please check, whether a malformed UTF-8 sequence is (1) represented at
+all, (2) represented by exactly one single replacement character (or
+equivalent signal), and (3) the following quotation mark after an
+illegal UTF-8 sequence is correctly displayed, i.e. proper
+resynchronization takes place immageately after any malformed
+sequence. This file says "THE END" in the last line, so if you don't
+see that, your decoder crashed somehow before, which should always be
+cause for concern.
+
+All lines in this file are exactly 79 characters long (plus the line
+feed). In addition, all lines end with "|", except for the two test
+lines 2.1.1 and 2.2.1, which contain non-printable ASCII controls
+U+0000 and U+007F. If you display this file with a fixed-width font,
+these "|" characters should all line up in column 79 (right margin).
+This allows you to test quickly, whether your UTF-8 decoder finds the
+correct number of characters in every line, that is whether each
+malformed sequences is replaced by a single replacement character.
+
+Note that as an alternative to the notion of malformed sequence used
+here, it is also a perfectly acceptable (and in some situations even
+preferable) solution to represent each individual byte of a malformed
+sequence by a replacement character. If you follow this strategy in
+your decoder, then please ignore the "|" column.
+
+
+Here come the tests:                                                          |
+                                                                              |
+1  Some correct UTF-8 text                                                    |
+                                                                              |
+(The codepoints for this test are:                                            |
+  U+03BA U+1F79 U+03C3 U+03BC U+03B5  --ryan.)                                |
+                                                                              |
+You should see the Greek word 'kosme':       "κόσμε"                          |
+                                                                              |
+                                                                              |
+2  Boundary condition test cases                                              |
+                                                                              |
+2.1  First possible sequence of a certain length                              |
+                                                                              |
+(byte zero skipped...there's a null added at the end of the test. --ryan.)    |
+                                                                              |
+2.1.2  2 bytes (U-00000080):        "€"                                       |
+2.1.3  3 bytes (U-00000800):        "ࠀ"                                       |
+2.1.4  4 bytes (U-00010000):        "𐀀"                                       |
+                                                                              |
+(5 and 6 byte sequences were made illegal in rfc3629. --ryan.)                |
+2.1.5  5 bytes (U-00200000):        "�����"                                       |
+2.1.6  6 bytes (U-04000000):        "������"                                       |
+                                                                              |
+2.2  Last possible sequence of a certain length                               |
+                                                                              |
+2.2.1  1 byte  (U-0000007F):        ""                                       |
+2.2.2  2 bytes (U-000007FF):        "߿"                                       |
+                                                                              |
+(Section 5.3.2 below calls this illegal. --ryan.)                             |
+2.2.3  3 bytes (U-0000FFFF):        "￿"                                       |
+                                                                              |
+(5 and 6 bytes sequences, and 4 bytes sequences > 0x10FFFF were made illegal  |
+ in rfc3629, so these next three should be replaced with a invalid            |
+ character codepoint. --ryan.)                                                |
+2.2.4  4 bytes (U-001FFFFF):        "����"                                       |
+2.2.5  5 bytes (U-03FFFFFF):        "�����"                                       |
+2.2.6  6 bytes (U-7FFFFFFF):        "������"                                       |
+                                                                              |
+2.3  Other boundary conditions                                                |
+                                                                              |
+2.3.1  U-0000D7FF = ed 9f bf = "퟿"                                            |
+2.3.2  U-0000E000 = ee 80 80 = ""                                            |
+2.3.3  U-0000FFFD = ef bf bd = "�"                                            |
+2.3.4  U-0010FFFF = f4 8f bf bf = "􏿿"                                         |
+                                                                              |
+(This one is bogus in rfc3629. --ryan.)                                       |
+2.3.5  U-00110000 = f4 90 80 80 = "����"                                         |
+                                                                              |
+3  Malformed sequences                                                        |
+                                                                              |
+3.1  Unexpected continuation bytes                                            |
+                                                                              |
+Each unexpected continuation byte should be separately signalled as a         |
+malformed sequence of its own.                                                |
+                                                                              |
+3.1.1  First continuation byte 0x80: "�"                                      |
+3.1.2  Last  continuation byte 0xbf: "�"                                      |
+                                                                              |
+3.1.3  2 continuation bytes: "��"                                             |
+3.1.4  3 continuation bytes: "���"                                            |
+3.1.5  4 continuation bytes: "����"                                           |
+3.1.6  5 continuation bytes: "�����"                                          |
+3.1.7  6 continuation bytes: "������"                                         |
+3.1.8  7 continuation bytes: "�������"                                        |
+                                                                              |
+3.1.9  Sequence of all 64 possible continuation bytes (0x80-0xbf):            |
+                                                                              |
+   "����������������                                                          |
+    ����������������                                                          |
+    ����������������                                                          |
+    ����������������"                                                         |
+                                                                              |
+3.2  Lonely start characters                                                  |
+                                                                              |
+3.2.1  All 32 first bytes of 2-byte sequences (0xc0-0xdf),                    |
+       each followed by a space character:                                    |
+                                                                              |
+   "� � � � � � � � � � � � � � � �                                           |
+    � � � � � � � � � � � � � � � � "                                         |
+                                                                              |
+3.2.2  All 16 first bytes of 3-byte sequences (0xe0-0xef),                    |
+       each followed by a space character:                                    |
+                                                                              |
+   "� � � � � � � � � � � � � � � � "                                         |
+                                                                              |
+3.2.3  All 8 first bytes of 4-byte sequences (0xf0-0xf7),                     |
+       each followed by a space character:                                    |
+                                                                              |
+   "� � � � � � � � "                                                         |
+                                                                              |
+3.2.4  All 4 first bytes of 5-byte sequences (0xf8-0xfb),                     |
+       each followed by a space character:                                    |
+                                                                              |
+   "� � � � "                                                                 |
+                                                                              |
+3.2.5  All 2 first bytes of 6-byte sequences (0xfc-0xfd),                     |
+       each followed by a space character:                                    |
+                                                                              |
+   "� � "                                                                     |
+                                                                              |
+3.3  Sequences with last continuation byte missing                            |
+                                                                              |
+All bytes of an incomplete sequence should be signalled as a single           |
+malformed sequence, i.e., you should see only a single replacement            |
+character in each of the next 10 tests. (Characters as in section 2)          |
+                                                                              |
+3.3.1  2-byte sequence with last byte missing (U+0000):     "�"               |
+3.3.2  3-byte sequence with last byte missing (U+0000):     "��"               |
+3.3.3  4-byte sequence with last byte missing (U+0000):     "���"               |
+3.3.4  5-byte sequence with last byte missing (U+0000):     "����"               |
+3.3.5  6-byte sequence with last byte missing (U+0000):     "�����"               |
+3.3.6  2-byte sequence with last byte missing (U-000007FF): "�"               |
+3.3.7  3-byte sequence with last byte missing (U-0000FFFF): "�"               |
+3.3.8  4-byte sequence with last byte missing (U-001FFFFF): "���"               |
+3.3.9  5-byte sequence with last byte missing (U-03FFFFFF): "����"               |
+3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): "�����"               |
+                                                                              |
+3.4  Concatenation of incomplete sequences                                    |
+                                                                              |
+All the 10 sequences of 3.3 concatenated, you should see 10 malformed         |
+sequences being signalled:                                                    |
+                                                                              |
+   "�����������������������������"                                                               |
+                                                                              |
+3.5  Impossible bytes                                                         |
+                                                                              |
+The following two bytes cannot appear in a correct UTF-8 string               |
+                                                                              |
+3.5.1  fe = "�"                                                               |
+3.5.2  ff = "�"                                                               |
+3.5.3  fe fe ff ff = "����"                                                   |
+                                                                              |
+4  Overlong sequences                                                         |
+                                                                              |
+The following sequences are not malformed according to the letter of          |
+the Unicode 2.0 standard. However, they are longer then necessary and         |
+a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8         |
+decoder" should reject them just like malformed sequences for two             |
+reasons: (1) It helps to debug applications if overlong sequences are         |
+not treated as valid representations of characters, because this helps        |
+to spot problems more quickly. (2) Overlong sequences provide                 |
+alternative representations of characters, that could maliciously be          |
+used to bypass filters that check only for ASCII characters. For              |
+instance, a 2-byte encoded line feed (LF) would not be caught by a            |
+line counter that counts only 0x0a bytes, but it would still be               |
+processed as a line feed by an unsafe UTF-8 decoder later in the              |
+pipeline. From a security point of view, ASCII compatibility of UTF-8         |
+sequences means also, that ASCII characters are *only* allowed to be          |
+represented by ASCII bytes in the range 0x00-0x7f. To ensure this             |
+aspect of ASCII compatibility, use only "safe UTF-8 decoders" that            |
+reject overlong UTF-8 sequences for which a shorter encoding exists.          |
+                                                                              |
+4.1  Examples of an overlong ASCII character                                  |
+                                                                              |
+With a safe UTF-8 decoder, all of the following five overlong                 |
+representations of the ASCII character slash ("/") should be rejected         |
+like a malformed UTF-8 sequence, for instance by substituting it with         |
+a replacement character. If you see a slash below, you do not have a          |
+safe UTF-8 decoder!                                                           |
+                                                                              |
+4.1.1 U+002F = c0 af             = "��"                                        |
+4.1.2 U+002F = e0 80 af          = "���"                                        |
+4.1.3 U+002F = f0 80 80 af       = "����"                                        |
+4.1.4 U+002F = f8 80 80 80 af    = "�����"                                        |
+4.1.5 U+002F = fc 80 80 80 80 af = "������"                                        |
+                                                                              |
+4.2  Maximum overlong sequences                                               |
+                                                                              |
+Below you see the highest Unicode value that is still resulting in an         |
+overlong sequence if represented with the given number of bytes. This         |
+is a boundary test for safe UTF-8 decoders. All five characters should        |
+be rejected like malformed UTF-8 sequences.                                   |
+                                                                              |
+4.2.1  U-0000007F = c1 bf             = "��"                                   |
+4.2.2  U-000007FF = e0 9f bf          = "���"                                   |
+4.2.3  U-0000FFFF = f0 8f bf bf       = "����"                                   |
+4.2.4  U-001FFFFF = f8 87 bf bf bf    = "�����"                                   |
+4.2.5  U-03FFFFFF = fc 83 bf bf bf bf = "������"                                   |
+                                                                              |
+4.3  Overlong representation of the NUL character                             |
+                                                                              |
+The following five sequences should also be rejected like malformed           |
+UTF-8 sequences and should not be treated like the ASCII NUL                  |
+character.                                                                    |
+                                                                              |
+4.3.1  U+0000 = c0 80             = "��"                                       |
+4.3.2  U+0000 = e0 80 80          = "���"                                       |
+4.3.3  U+0000 = f0 80 80 80       = "����"                                       |
+4.3.4  U+0000 = f8 80 80 80 80    = "�����"                                       |
+4.3.5  U+0000 = fc 80 80 80 80 80 = "������"                                       |
+                                                                              |
+5  Illegal code positions                                                     |
+                                                                              |
+The following UTF-8 sequences should be rejected like malformed               |
+sequences, because they never represent valid ISO 10646 characters and        |
+a UTF-8 decoder that accepts them might introduce security problems           |
+comparable to overlong UTF-8 sequences.                                       |
+                                                                              |
+5.1 Single UTF-16 surrogates                                                  |
+                                                                              |
+5.1.1  U+D800 = ed a0 80 = "���"                                                |
+5.1.2  U+DB7F = ed ad bf = "���"                                                |
+5.1.3  U+DB80 = ed ae 80 = "���"                                                |
+5.1.4  U+DBFF = ed af bf = "���"                                                |
+5.1.5  U+DC00 = ed b0 80 = "���"                                                |
+5.1.6  U+DF80 = ed be 80 = "���"                                                |
+5.1.7  U+DFFF = ed bf bf = "���"                                                |
+                                                                              |
+5.2 Paired UTF-16 surrogates                                                  |
+                                                                              |
+5.2.1  U+D800 U+DC00 = ed a0 80 ed b0 80 = "������"                               |
+5.2.2  U+D800 U+DFFF = ed a0 80 ed bf bf = "������"                               |
+5.2.3  U+DB7F U+DC00 = ed ad bf ed b0 80 = "������"                               |
+5.2.4  U+DB7F U+DFFF = ed ad bf ed bf bf = "������"                               |
+5.2.5  U+DB80 U+DC00 = ed ae 80 ed b0 80 = "������"                               |
+5.2.6  U+DB80 U+DFFF = ed ae 80 ed bf bf = "������"                               |
+5.2.7  U+DBFF U+DC00 = ed af bf ed b0 80 = "������"                               |
+5.2.8  U+DBFF U+DFFF = ed af bf ed bf bf = "������"                               |
+                                                                              |
+5.3 Other illegal code positions                                              |
+                                                                              |
+5.3.1  U+FFFE = ef bf be = "￾"                                                |
+5.3.2  U+FFFF = ef bf bf = "￿"                                                |
+                                                                              |
+THE END                                                                       |
+