u 0.5.0 → 1.0.0

data/ext/u/u_collation_key.c

@@ -0,0 +1,132 @@
+#include "extconf.h"
+#include <assert.h>
+#include <errno.h>
+#define __USE_XOPEN2K8 1
+#include <locale.h>
+#ifdef HAVE_XLOCALE_H
+#  include <xlocale.h>
+#endif
+#define __USE_XOPEN2K 1
+#include <langinfo.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#ifndef HAVE_STRXFRM_L
+static inline size_t
+strxfrm_l(char *restrict s1, const char *restrict s2, size_t n,
+          UNUSED(locale_t loc))
+{
+        return strxfrm(s1, s2, n);
+}
+#endif
+#ifndef HAVE_NL_LANGINFO_L
+static inline char *
+nl_langinfo_l(nl_item item, UNUSED(locale_t loc))
+{
+        return nl_langinfo(item);
+}
+#endif
+
+#include "u.h"
+#include "private.h"
+
+static inline const char *
+codeset(locale_t locale)
+{
+        return locale == NULL ?
+                nl_langinfo(CODESET) :
+                nl_langinfo_l(CODESET, locale);
+}
+
+static inline size_t
+transform(char *result, const char *string, size_t n, locale_t locale)
+{
+        return locale == NULL ?
+                strxfrm(result, string, n) :
+                strxfrm_l(result, string, n, locale);
+}
+
+static size_t
+ckey(char *result, size_t m, const char *string, size_t n, locale_t locale)
+{
+        char saved_sentinel = string[n];
+        ((char *)string)[n] = '\0';
+        size_t l = 0;
+        const char *p = string;
+        const char *end = string + n + 1;
+        while (true) {
+                errno = 0;
+                size_t k = m > l ?
+                        transform(result + l, p, m - l, locale) :
+                        transform(NULL, p, 0, locale);
+                if (errno != 0)
+                        break;
+                l += k;
+                p += strlen(p) + 1;
+                if (p == end)
+                        break;
+                if (m > l)
+                        result[l] = '\0';
+                l++;
+        }
+        ((char *)string)[n] = saved_sentinel;
+        return l;
+}
+
+static size_t
+recode_ckey(char *result, size_t m, const char *string, size_t n,
+            locale_t locale, const char *cs)
+{
+        char buf[2048];
+        errno = 0;
+        size_t n_recoded = u_recode(buf, sizeof(buf), string, n, cs);
+        if (errno != 0)
+                return 0;
+        if (n_recoded < sizeof(buf))
+                return ckey(result, m, buf, n_recoded, locale);
+        char *recoded = malloc(n_recoded + 1);
+        if (recoded == NULL)
+                return 0;
+        u_recode(recoded, n_recoded + 1, string, n, cs);
+        size_t n_key = ckey(result, m, recoded, n_recoded, locale);
+        free(recoded);
+        return n_key;
+}
+
+size_t
+u_collation_key(char *result, size_t m, const char *string, size_t n,
+                const char *locale)
+{
+	assert(string != NULL);
+        assert(result != NULL || m == 0);
+        locale_t l = NULL;
+        if (locale != NULL)
+                l = newlocale(LC_COLLATE_MASK | LC_CTYPE_MASK, locale, NULL);
+        const char *cs = codeset(l);
+        size_t r = strcmp(cs, "UTF-8") != 0 ?
+                recode_ckey(result, m, string, n, l, cs) :
+                ckey(result, m, string, n, l);
+        if (l != NULL)
+                freelocale(l);
+        return r;
+}
+
+size_t
+u_normalized_collation_key(char *result, size_t m, const char *string, size_t n,
+                           const char *locale)
+{
+	assert(string != NULL);
+        assert(result != NULL || m == 0);
+        char buf[2048];
+        size_t n_normalized = u_normalize(buf, sizeof(buf), string, n, U_NORMALIZATION_FORM_KC);
+        if (n_normalized < sizeof(buf))
+                return u_collation_key(result, m, buf, n_normalized, locale);
+        char *normalized = malloc(n_normalized + 1);
+        if (normalized == NULL)
+                return 0;
+        u_normalize(normalized, n_normalized + 1, string, n, U_NORMALIZATION_FORM_KC);
+        size_t n_key = u_collation_key(result, m, normalized, n_normalized, locale);
+        free(normalized);
+        return n_key;
+}

data/ext/u/u_decode.c

@@ -0,0 +1,156 @@
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#include "u.h"
+#include "private.h"
+
+
+// The dfa table and decode function is © 2008–2010 Björn Höhrmann
+// <bjoern@hoehrmann.de>.  See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+// for details.
+
+enum {
+        ACCEPT = 0,
+        REJECT = 12
+};
+
+static const uint8_t dfa[] = {
+        // The first part of the table maps bytes to character classes to
+        // reduce the size of the transition table and create bitmasks.
+        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+        1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+        7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+        8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+       10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
+
+        // The second part is a transition table that maps a combination
+        // of a state of the automaton and a character class to a state.
+        0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
+       12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
+       12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
+       12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
+       12,36,12,12,12,12,12,12,12,12,12,12,
+};
+
+// This reversal of ‹dfa› is © 2013 Nikolai Weibull.
+static const uint8_t dfa_r[] = {
+        // The first part of the table maps bytes to character classes to
+        // reduce the size of the transition table and create bitmasks.
+        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+        1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+        7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+        8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+       10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
+
+        // The second part is a transition table that maps a combination
+        // of a state of the automaton and a character class to a state.
+        0,24,12,12,12,12,12,24,12,24,12,12, 12,12,12,12,12,12,12,12,12,12,12,12,
+       12,36, 0,12,12,12,12,48,12,36,12,12, 12,60,12, 0, 0,12,12,72,12,72,12,12,
+       12,60,12, 0,12,12,12,72,12,72, 0,12, 12,12,12,12,12, 0, 0,12,12,12,12,12,
+       12,12,12,12,12,12, 0,12,12,12,12, 0,
+};
+
+static inline uint32_t
+decode(uint32_t *state, uint32_t *c, uint32_t b)
+{
+        uint32_t type = dfa[b];
+        *c = *state != ACCEPT ? (*c << 6) | (b & 0x3f) : (0xff >> type) & b;
+        *state = dfa[256 + *state + type];
+        return *state;
+}
+
+#define REPLACEMENT_CHARACTER ((uint32_t)0xfffd)
+
+uint32_t
+u_decode(const char **q, const char *u, const char *end)
+{
+        assert(u < end);
+        uint32_t c, state = ACCEPT;
+        const unsigned char *p;
+        for (p = (const unsigned char *)u; p < (const unsigned char *)end; p++)
+                switch (decode(&state, &c, *p)) {
+                case ACCEPT:
+                        *q = (const char *)p + 1;
+                        return c;
+                case REJECT:
+                        *q = (const char *)p + 1;
+                        return REPLACEMENT_CHARACTER;
+                }
+        *q = (const char *)p;
+        return REPLACEMENT_CHARACTER;
+}
+
+int
+u_decode_n(uint32_t *result, const char *u, size_t n)
+{
+        const char *q;
+        *result = u_decode(&q, u, u + n);
+        return (int)(q - u);
+}
+
+static inline uint32_t
+decode_r(uint32_t *state, uint32_t *c, uint32_t b, int i)
+{
+        uint32_t type = dfa_r[b];
+        *state = dfa_r[256 + *state + type];
+        *c |= (*state != ACCEPT ? b & 0x3f : (0xff >> type) & b) << (6*i);
+        return *state;
+}
+
+uint32_t
+u_decode_r(const char **p, const char *begin, const char *u)
+{
+        assert(begin < u);
+        uint32_t c = 0, state = ACCEPT;
+        int i = 0;
+        const unsigned char *q;
+        for (q = (const unsigned char *)u - 1; (const unsigned char *)begin <= q; q--, i++)
+                switch (decode_r(&state, &c, *q, i)) {
+                case ACCEPT:
+                        *p = (const char *)q;
+                        return c;
+                case REJECT:
+                        *p = (const char *)q;
+                        return REPLACEMENT_CHARACTER;
+                }
+        *p = (const char *)begin;
+        return REPLACEMENT_CHARACTER;
+}
+
+static inline uint32_t
+validate(uint32_t *state, uint32_t b)
+{
+        uint32_t type = dfa[b];
+        return *state = dfa[256 + *state + type];
+}
+
+bool
+u_valid(const char *u, size_t n, const char **end)
+{
+        uint32_t state = ACCEPT;
+        const unsigned char *p = (const unsigned char *)u;
+        const unsigned char *o = p;
+        const unsigned char *q = p + n;
+        for ( ; p < q; p++)
+                switch (validate(&state, *p)) {
+                case ACCEPT:
+                        o = p;
+                        break;
+                case REJECT:
+                reject:
+                        if (end != NULL)
+                                *end = (const char *)o;
+                        return false;
+                }
+        if (p == q)
+                return state == ACCEPT;
+        goto reject;
+}

data/ext/u/u_downcase.c

@@ -0,0 +1,201 @@
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "u.h"
+
+#include <assert.h>
+
+#include <string.h>
+#include "output.h"
+
+#include "data/constants.h"
+#include "attributes.h"
+#include "u_locale.h"
+#include "titled.h"
+#include "case.h"
+#include "private.h"
+
+#define LATIN_CAPITAL_LETTER_I ((uint32_t)0x0049)
+#define LATIN_CAPITAL_LETTER_J ((uint32_t)0x004a)
+#define LATIN_SMALL_LETTER_I ((uint32_t)0x0069)
+#define LATIN_CAPITAL_LETTER_I_WITH_GRAVE ((uint32_t)0x00cc)
+#define LATIN_CAPITAL_LETTER_I_WITH_ACUTE ((uint32_t)0x00cd)
+#define LATIN_CAPITAL_LETTER_I_WITH_TILDE ((uint32_t)0x0128)
+#define LATIN_CAPITAL_LETTER_I_WITH_OGONEK ((uint32_t)0x012e)
+#define LATIN_CAPITAL_LETTER_I_WITH_DOT_ABOVE ((uint32_t)0x0130)
+#define LATIN_SMALL_LETTER_DOTLESS_I ((uint32_t)0x0131)
+#define COMBINING_GRAVE_ACCENT ((uint32_t)0x0300)
+#define COMBINING_ACUTE_ACCENT ((uint32_t)0x0301)
+#define COMBINING_TILDE ((uint32_t)0x0303)
+#define COMBINING_DOT_ABOVE ((uint32_t)0x0307)
+#define GREEK_CAPITAL_LETTER_SIGMA ((uint32_t)0x03a3)
+#define GREEK_SMALL_LETTER_FINAL_SIGMA ((uint32_t)0x03c2)
+#define GREEK_SMALL_LETTER_SIGMA ((uint32_t)0x03c3)
+
+
+static inline bool
+is_final_sigma(const char *string, const char *p, const char *q, const char *end)
+{
+        if (p == string)
+                return false;
+        uint32_t c;
+        while (q < end) {
+                c = u_decode(&q, q, end);
+                if (u_char_iscaseignorable(c))
+                        continue;
+                if (u_char_iscased(c))
+                        return false;
+                break;
+        }
+        while (string < p) {
+                c = u_decode_r(&p, string, p);
+                if (u_char_iscaseignorable(c))
+                        continue;
+                if (u_char_iscased(c))
+                        return true;
+                return false;
+        }
+        return false;
+}
+
+static inline bool
+has_more_above(const char *q, const char *end)
+{
+        while (q < end) {
+		switch (u_char_canonical_combining_class(u_decode(&q, q, end))) {
+                case U_CANONICAL_COMBINING_CLASS_ABOVE:
+			return true;
+                case U_CANONICAL_COMBINING_CLASS_NOT_REORDERED:
+			return false;
+                default:
+                        break;
+                }
+	}
+	return false;
+}
+
+static bool
+downcase_lithuanian_i(uint32_t c, uint32_t combiner, struct output *output)
+{
+        output_char(output, c);
+        output_char(output, COMBINING_DOT_ABOVE);
+        if (combiner != '\0')
+                output_char(output, combiner);
+        return true;
+}
+
+static inline bool
+downcase_lithuanian(uint32_t c, const char *q, const char *end,
+                    struct output *output)
+{
+        switch (c) {
+        case LATIN_CAPITAL_LETTER_I:
+        case LATIN_CAPITAL_LETTER_J:
+        case LATIN_CAPITAL_LETTER_I_WITH_OGONEK:
+                if (!has_more_above(q, end))
+                        return false;
+                return downcase_lithuanian_i(u_char_downcase(c), '\0', output);
+        case LATIN_CAPITAL_LETTER_I_WITH_GRAVE:
+                return downcase_lithuanian_i(LATIN_SMALL_LETTER_I,
+                                             COMBINING_GRAVE_ACCENT, output);
+        case LATIN_CAPITAL_LETTER_I_WITH_ACUTE:
+                return downcase_lithuanian_i(LATIN_SMALL_LETTER_I,
+                                             COMBINING_ACUTE_ACCENT, output);
+        case LATIN_CAPITAL_LETTER_I_WITH_TILDE:
+                return downcase_lithuanian_i(LATIN_SMALL_LETTER_I,
+                                             COMBINING_TILDE, output);
+        default:
+                return false;
+        }
+}
+
+static inline bool
+is_before_dot(const char *q, const char *end)
+{
+        while (q < end) {
+                uint32_t c = u_decode(&q, q, end);
+                if (c == COMBINING_DOT_ABOVE)
+                        return true;
+                switch (u_char_canonical_combining_class(c)) {
+                case U_CANONICAL_COMBINING_CLASS_ABOVE:
+                case U_CANONICAL_COMBINING_CLASS_NOT_REORDERED:
+                        return false;
+                default:
+                        break;
+                }
+	}
+        return false;
+}
+
+static bool
+is_i(uint32_t c)
+{
+        return c == LATIN_CAPITAL_LETTER_I;
+}
+
+static inline bool
+downcase_turkic(uint32_t c,
+                const char *string, const char *p, const char *q, const char *end,
+                struct output *output)
+{
+        switch (c) {
+        case LATIN_CAPITAL_LETTER_I:
+                output_char(output,
+                            is_before_dot(q, end) ?
+                            LATIN_SMALL_LETTER_I :
+                            LATIN_SMALL_LETTER_DOTLESS_I);
+                return true;
+        case LATIN_CAPITAL_LETTER_I_WITH_DOT_ABOVE:
+                case_simple(LATIN_CAPITAL_LETTER_I,
+                            U_GENERAL_CATEGORY_LETTER_UPPERCASE,
+                            false, false, output);
+                return true;
+        case COMBINING_DOT_ABOVE:
+                if (!is_after(string, p, is_i))
+                        output_char(output, COMBINING_DOT_ABOVE);
+                return true;
+        default:
+                return false;
+        }
+}
+
+const char *
+_u_downcase_step(const char *string, const char *p, const char *end,
+                 enum locale locale, struct output *output)
+{
+        const char *q;
+        uint32_t c = u_decode(&q, p, end);
+        enum u_general_category gc;
+        if (c == GREEK_CAPITAL_LETTER_SIGMA)
+                output_char(output,
+                            is_final_sigma(string, p, q, end) ?
+                            GREEK_SMALL_LETTER_FINAL_SIGMA :
+                            GREEK_SMALL_LETTER_SIGMA);
+        else if (locale == LOCALE_LITHUANIAN &&
+                 downcase_lithuanian(c, q, end, output))
+                ;
+        else if (locale == LOCALE_TURKIC &&
+                 downcase_turkic(c, string, p, q, end, output))
+                ;
+        else if (IS(gc = u_char_general_category(c),
+                    OR(U_GENERAL_CATEGORY_LETTER_UPPERCASE,
+                       OR(U_GENERAL_CATEGORY_LETTER_TITLECASE, 0))))
+                case_simple(c, gc, false, false, output);
+        else
+                output_string(output, p, q - p);
+        return q;
+}
+
+size_t
+u_downcase(char *result, size_t m, const char *u, size_t n,
+           const char *locale)
+{
+	assert(u != NULL);
+        assert(result != NULL || m == 0);
+	enum locale l = _u_locale_from_string(locale);
+        const char *end = u + n;
+        struct output o = OUTPUT_INIT(result, m);
+        for (const char *p = u; p < end; )
+                p = _u_downcase_step(u, p, end, l, &o);
+        return output_finalize(&o);
+}