u 0.5.0 → 1.0.0

data/ext/{encoding/character/utf-8 → u}/private.c

@@ -1,38 +1,34 @@
-/*
- * contents: Private functions used by the UTF-8 character-encoding library.
- *
- * Copyright © 2007 Nikolai Weibull <now@bitwi.se>
- */
-
 #include <ruby.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
 #include <stdlib.h>
 
-#include "unicode.h"
+#include "u.h"
 
 #include "private.h"
 
 /* Lookup C in the sorted TABLE using binary search.  TABLE consists of N
  * entries, where each entry is SIZEOF_ENTRY bytes in size and the first
- * component is a unichar of size SIZEOF_CHAR.  If C is found in TABLE, its
+ * component is a uint32_t of size SIZEOF_CHAR.  If C is found in TABLE, its
  * index is stored in INDEX and true is returned.  Otherwise, false is returned
  * and INDEX is left untouched. */
 bool
-binary_search_unicode_table(const void *table, size_t n, size_t sizeof_entry, size_t sizeof_char, unichar c, int *index)
+binary_search_unicode_table(const void *table, size_t n, size_t sizeof_entry, size_t sizeof_char, uint32_t c, size_t *index)
 {
-#define ENTRY(index) ((unichar)(*(unichar *)((const char *)table + ((index) * sizeof_entry))) & char_mask)
+#define ENTRY(index) (*(uint32_t *)(void *)((const char *)table + ((index) * sizeof_entry)) & char_mask)
 
-	int begin = 0;
-        int end = n - 1;
-        int middle;
+	size_t begin = 0;
+        size_t end = n - 1;
+        size_t middle;
 
         /* This is ugly, but not all tables use unichars as their lookup
          * character.  The casefold table, for example, uses uint16_t-sized
          * characters.  To only get the interesting part of our table entry
          * we’ll have to mask the retrieved value. */
-        int char_mask = (1 << (8 * sizeof_char)) - 1;
+        uint32_t char_mask = sizeof_char < sizeof(uint32_t) ?
+                ((uint32_t)1 << (CHAR_BIT * sizeof_char)) - 1 :
+                (uint32_t)-1;
 
         /* Drop out early if we know for certain that C can’t be in the
          * decomposition table. */
@@ -42,7 +38,7 @@ binary_search_unicode_table(const void *table, size_t n, size_t sizeof_entry, si
         while (begin <= end) {
                 middle = binary_search_middle_of(begin, end);
 
-                unichar probe = ENTRY(middle);
+                uint32_t probe = ENTRY(middle);
                 if (c < probe)
                         end = middle - 1;
                 else if (c > probe)

data/ext/u/private.h

@@ -0,0 +1,58 @@
+#define IS(category, class) (((unsigned int)1 << (category)) & (class))
+#define OR(class, rest) (((unsigned int)1 << (class)) | (rest))
+
+#define P_WITHIN_STR(p, end, use_end) \
+        ((use_end) ? (p) < (end) : *(p) != '\0')
+
+#define lengthof(ary)   (sizeof(ary) / sizeof((ary)[0]))
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#  ifdef U_COMPILATION
+#    define U_EXTERN __declspec(dllexport) extern
+#  else
+#    define U_EXTERN __declspec(dllimport) extern
+#  endif
+#elif __GNUC__ >= 4
+#  define U_EXTERN __attribute__((visibility("default"))) extern
+#else
+#  define U_EXTERN extern
+#endif
+
+#if defined(__GNUC__) && __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 4)
+#  define PRINTF(format_index, first_argument_index) \
+        __attribute__((format(printf, format_index, first_argument_index)))
+#  define UNUSED(u) \
+        __attribute__((__unused__)) u
+#else
+#  define PRINTF(format, arguments)
+#  define UNUSED(u) u
+#endif
+
+#if defined(__GNUC__) && __GNUC__ > 2 && defined(__OPTIMIZE__)
+#  define BOOLEAN_EXPR(expr) __extension__({ \
+        int _boolean_var_; \
+        if (expr) \
+                _boolean_var_ = 1; \
+        else \
+                _boolean_var_ = 0; \
+        _boolean_var_; \
+})
+#  define LIKELY(expr) (__builtin_expect(BOOLEAN_EXPR(expr), 1))
+#  define UNLIKELY(expr) (__builtin_expect(BOOLEAN_EXPR(expr), 0))
+#else
+#  define LIKELY(expr) (expr)
+#  define UNLIKELY(expr) (expr)
+#endif
+
+#define binary_search_middle_of(begin, end)     \
+        (((unsigned)((begin) + (end))) >> 1)
+
+#define unicode_table_lookup(table, c, index)    \
+        binary_search_unicode_table(table, lengthof(table), sizeof((table)[0]), sizeof((table)[0].ch), c, index)
+
+bool binary_search_unicode_table(const void *table,
+                                 size_t n,
+                                 size_t sizeof_entry,
+                                 size_t sizeof_char,
+                                 uint32_t c,
+                                 size_t *index);

data/ext/u/rb_includes.h

@@ -0,0 +1,10 @@
+#include <ruby.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "extconf.h"
+#include "u.h"
+#include "private.h"
+#include "rb_private.h"
+#include "rb_u_string.h"

data/ext/u/rb_private.c

@@ -0,0 +1,98 @@
+#include "rb_includes.h"
+#ifdef HAVE_RUBY_ENCODING_H
+#  include <ruby/encoding.h>
+#endif
+#include <errno.h>
+
+#ifndef HAVE_RB_MEMHASH
+int
+rb_memhash(const char *string, long length)
+{
+        const char *p = string;
+        const char *end = string + length;
+        int hash = 0;
+
+        while (p < end) {
+                hash = hash * 65599 + *p;
+                p++;
+        }
+
+        return hash + (hash >> 5);
+}
+#endif
+
+long
+rb_u_memsearch(const void *a, long a_n, const void *b, long b_n)
+{
+#ifdef HAVE_RUBY_ENCODING_H
+        return rb_memsearch(a, a_n, b, b_n, rb_utf8_encoding());
+#else
+        return rb_memsearch(a, a_n, b, b_n);
+#endif
+}
+
+static VALUE PRINTF(1, 0)
+format_message(const char *format, va_list args)
+{
+#ifdef HAVE_RUBY_ENCODING_H
+        return rb_enc_vsprintf(rb_utf8_encoding(), format, args);
+#else
+#  ifdef HAVE_RB_VSPRINTF
+        return rb_vsprintf(format, args);
+#  else
+        char buf[2048];
+        int n = vsnprintf(buf, sizeof(buf), format, args);
+        return rb_str_new(buf, n);
+#  endif
+#endif
+}
+
+void
+rb_u_raise(VALUE exception, const char *format, ...)
+{
+        va_list args;
+        va_start(args, format);
+        VALUE message = format_message(format, args);
+        va_end(args);
+        rb_exc_raise(rb_exc_new3(exception, message));
+}
+
+void
+rb_u_raise_errno(int number, const char *format, ...)
+{
+        va_list args;
+        va_start(args, format);
+#ifdef HAVE_RUBY_ENCODING_H
+        VALUE message = format_message(format, args);
+        va_end(args);
+        rb_syserr_fail_str(number, message);
+#else
+        char buf[2048];
+        vsnprintf(buf, sizeof(buf), format, args);
+        errno = number;
+        va_end(args);
+        rb_sys_fail(buf);
+#endif
+}
+
+VALUE
+rb_u_str_new(const char *string, long length)
+{
+#ifdef HAVE_RUBY_ENCODING_H
+        return rb_enc_str_new(string, length, rb_utf8_encoding());
+#else
+        return rb_str_new(string, length);
+#endif
+}
+
+VALUE
+rb_u_str_buf_new(long length)
+{
+#ifdef HAVE_RUBY_ENCODING_H
+        VALUE buffer = rb_str_buf_new(length);
+        rb_enc_associate(buffer, rb_utf8_encoding());
+        return buffer;
+#else
+        return rb_str_buf_new(length);
+#endif
+}

data/ext/u/rb_private.h

@@ -0,0 +1,67 @@
+#if __GNUC__ >= 4
+#  define RB_U_NULL_TERMINATED(parameter) __attribute__((__sentinel__(parameter)))
+#else
+#  define RB_U_NULL_TERMINATED(parameter)
+#endif
+
+void need_at_least_n_arguments(int argc, int n);
+
+void need_m_to_n_arguments(int argc, int m, int n);
+
+void *_rb_u_guarded_alloc(size_t n, ...) RB_U_NULL_TERMINATED(0);
+
+int rb_u_char_to_u(uint32_t c, char *result);
+
+void rb_u_validate(const char *string, long length);
+
+VALUE _rb_u_character_test(VALUE string, bool (*test)(uint32_t));
+
+VALUE _rb_u_string_test_locale(int argc, VALUE *argv, VALUE self,
+                               size_t convert(char *, size_t, const char *, size_t,
+                                              const char *));
+
+VALUE _rb_u_string_convert(VALUE self,
+                           size_t convert(char *, size_t, const char *, size_t));
+VALUE _rb_u_string_convert_locale(int argc, VALUE *argv, VALUE self,
+                                  size_t convert(char *, size_t, const char *,
+                                                 size_t, const char *),
+                                  const char *lc_env);
+
+VALUE _rb_u_string_property(VALUE self, const char *name,
+                            int unknown, int property(uint32_t),
+                            VALUE tosym(int));
+
+enum u_normalization_form _rb_u_symbol_to_normalization_form(VALUE symbol);
+
+VALUE rb_u_pattern_argument(VALUE pattern, bool quote);
+
+long rb_u_string_index_regexp(VALUE self, const char *begin, VALUE regex, bool reverse);
+
+#ifndef HAVE_RB_ERRINFO
+#  define rb_errinfo() (ruby_errinfo)
+#endif
+
+#ifndef HAVE_RB_MEMHASH
+int rb_memhash(const char *string, long length);
+#endif
+
+#ifndef RETURN_SIZED_ENUMERATOR
+#  define RETURN_SIZED_ENUMERATOR(self, argc, argv, size) \
+        RETURN_ENUMERATOR(self, argc, argv)
+#endif
+
+#ifndef OBJ_UNTRUSTED
+#  define OBJ_UNTRUSTED(o) (false)
+#endif
+
+#ifndef OBJ_UNTRUST
+#  define OBJ_UNTRUST(o) do { } while (0)
+#endif
+
+long rb_u_memsearch(const void *a, long a_n, const void *b, long b_n);
+
+NORETURN(void rb_u_raise(VALUE exception, const char *format, ...)) PRINTF(2, 3);
+NORETURN(void rb_u_raise_errno(int number, const char *format, ...)) PRINTF(2, 3);
+
+VALUE rb_u_str_new(const char *string, long length);
+VALUE rb_u_str_buf_new(long length);

data/ext/u/rb_u.c

@@ -0,0 +1,251 @@
+/* -*- coding: utf-8 -*- */
+
+#include "extconf.h"
+#include <errno.h>
+#include <ruby.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <limits.h>
+#include "u.h"
+#include "private.h"
+#include "rb_private.h"
+#include "rb_u_buffer.h"
+#include "rb_u_string.h"
+
+void
+need_at_least_n_arguments(int argc, int n)
+{
+        if (argc < n)
+                rb_u_raise(rb_eArgError,
+                           "wrong number of arguments (%d for at least %d)",
+                           argc, n);
+}
+
+void
+need_m_to_n_arguments(int argc, int m, int n)
+{
+        if (argc < m || argc > n)
+                rb_u_raise(rb_eArgError,
+                           "wrong number of arguments (%d for %d..%d)",
+                           argc, m, n);
+}
+
+struct guarded_alloc_closure {
+        void *result;
+        size_t n;
+};
+
+static VALUE
+guarded_alloc(VALUE data)
+{
+        struct guarded_alloc_closure *closure = (struct guarded_alloc_closure *)data;
+        closure->result = (void *)ALLOC_N(char, closure->n);
+        return Qnil;
+}
+
+void *
+_rb_u_guarded_alloc(size_t n, ...)
+{
+        struct guarded_alloc_closure closure = { NULL, n };
+        int error;
+        rb_protect(guarded_alloc, (VALUE)&closure, &error);
+        if (error == 0)
+                return closure.result;
+        va_list args;
+        va_start(args, n);
+        void *previous;
+        while ((previous = va_arg(args, void *)) != NULL)
+                free(previous);
+        va_end(args);
+        rb_exc_raise(rb_errinfo());
+}
+
+int
+rb_u_char_to_u(uint32_t c, char *result)
+{
+        if (!u_char_isvalid(c))
+                rb_u_raise(rb_eArgError, "not a Unicode character: %#04x", c);
+
+        return u_char_to_u(c, result);
+}
+
+void
+rb_u_validate(const char *string, long length)
+{
+        const char *end;
+        if (!u_valid(string, length, &end))
+                rb_u_raise(rb_eArgError,
+                           "invalid byte sequence at byte %ld",
+                           end - string);
+}
+
+VALUE
+_rb_u_character_test(VALUE self, bool (*test)(uint32_t))
+{
+        const struct rb_u_string *s = RVAL2USTRING(self);
+        for (const char *p = USTRING_STR(s), *end = USTRING_END(s); p < end; )
+                if (!test(u_decode(&p, p, end)))
+                        return Qfalse;
+        return Qtrue;
+}
+
+VALUE
+_rb_u_string_test_locale(int argc, VALUE *argv, VALUE self,
+                         size_t convert(char *, size_t, const char *, size_t,
+                                        const char *))
+{
+        const char *locale = NULL;
+
+        VALUE rblocale;
+        if (rb_scan_args(argc, argv, "01", &rblocale) == 1)
+                locale = StringValuePtr(rblocale);
+
+        const struct rb_u_string *string = RVAL2USTRING(self);
+
+        size_t nfd_n = u_normalize(NULL, 0,
+                                   USTRING_STR(string), USTRING_LENGTH(string),
+                                   U_NORMALIZATION_FORM_D);
+        char *nfd = ALLOC_N(char, nfd_n + 1);
+        nfd_n = u_normalize(nfd, nfd_n + 1,
+                            USTRING_STR(string), USTRING_LENGTH(string),
+                            U_NORMALIZATION_FORM_D);
+
+        size_t converted_n = convert(NULL, 0, nfd, nfd_n, locale);
+        char *converted = _rb_u_guarded_alloc(converted_n + 1, nfd, NULL);
+        convert(converted, converted_n + 1, nfd, nfd_n, locale);
+
+        VALUE result = converted_n == nfd_n &&
+                memcmp(converted, nfd, nfd_n) == 0 ? Qtrue : Qfalse;
+
+        free(converted);
+        free(nfd);
+
+        return result;
+}
+
+VALUE
+_rb_u_string_convert(VALUE self,
+                     size_t convert(char *, size_t, const char *, size_t))
+{
+        const struct rb_u_string *string = RVAL2USTRING(self);
+
+        size_t n = convert(NULL, 0, USTRING_STR(string), USTRING_LENGTH(string));
+        char *converted = ALLOC_N(char, n + 1);
+        convert(converted, n + 1, USTRING_STR(string), USTRING_LENGTH(string));
+
+        return rb_u_string_new_c_own(self, converted, n);
+}
+
+static size_t
+try_convert(char *result, size_t m, const struct rb_u_string *string,
+            size_t convert(char *, size_t, const char *, size_t,
+                           const char *), const char *locale)
+{
+        errno = 0;
+        size_t n = convert(result, m, USTRING_STR(string), USTRING_LENGTH(string),
+                           locale);
+        if (errno != 0) {
+                free(result);
+                rb_u_raise_errno(errno, "can’t apply conversion");
+        }
+        return n;
+}
+
+VALUE
+_rb_u_string_convert_locale(int argc, VALUE *argv, VALUE self,
+                            size_t convert(char *, size_t, const char *, size_t,
+                                           const char *),
+                            const char *lc_env)
+{
+        const char *locale = NULL;
+
+        VALUE rblocale;
+        if (rb_scan_args(argc, argv, "01", &rblocale) == 1)
+                locale = StringValuePtr(rblocale);
+        else if (lc_env != NULL) {
+                const char * const env[] = { "LC_ALL", lc_env, "LANG", NULL };
+                for (const char * const *p = env; *p != NULL; p++)
+                        if ((locale = getenv(*p)) != NULL)
+                                break;
+        }
+
+        const struct rb_u_string *string = RVAL2USTRING(self);
+
+        size_t n = try_convert(NULL, 0, string, convert, locale);
+        char *converted = ALLOC_N(char, n + 1);
+        size_t m = try_convert(converted, n + 1, string, convert, locale);
+        if (m < n) {
+                char *t = REALLOC_N(converted, char, m + 1);
+                if (t != NULL)
+                        converted = t;
+                n = m;
+        }
+
+        return rb_u_string_new_c_own(self, converted, n);
+}
+
+VALUE
+_rb_u_string_property(VALUE self, const char *name, int unknown,
+                      int property(uint32_t), VALUE tosym(int))
+{
+        const struct rb_u_string *string = RVAL2USTRING(self);
+        const char *p = USTRING_STR(string);
+        const char *end = USTRING_END(string);
+        if (p == end)
+                return tosym(unknown);
+        int first = property(u_decode(&p, p, end));
+        while (p < end) {
+                int value = property(u_decode(&p, p, end));
+                if (value != first)
+                        rb_u_raise(rb_eArgError,
+                                   "string consists of characters with different %s values: :%s+, :%s",
+                                   name,
+                                   rb_id2name(SYM2ID(tosym(first))),
+                                   rb_id2name(SYM2ID(tosym(value))));
+        }
+        return tosym(first);
+}
+
+#define SYMBOL2MODE(symbol, mode, id) do { \
+        static ID id_##symbol; \
+        if (id_##symbol == 0) \
+                id_##symbol = rb_intern(#symbol); \
+        if (id == id_##symbol) \
+                return mode; \
+} while (0)
+
+enum u_normalization_form
+_rb_u_symbol_to_normalization_form(VALUE symbol)
+{
+        if (!SYMBOL_P(symbol)) {
+                VALUE inspected = rb_inspect(symbol);
+
+                rb_u_raise(rb_eTypeError,
+                           "not a symbol: %s",
+                           StringValuePtr(inspected));
+        }
+
+        ID id = SYM2ID(symbol);
+
+        SYMBOL2MODE(nfd, U_NORMALIZATION_FORM_D, id);
+        SYMBOL2MODE(nfc, U_NORMALIZATION_FORM_C, id);
+        SYMBOL2MODE(nfkd, U_NORMALIZATION_FORM_KD, id);
+        SYMBOL2MODE(nfkc, U_NORMALIZATION_FORM_KC, id);
+
+        rb_u_raise(rb_eArgError,
+                   "unknown normalization form: :%s",
+                   rb_id2name(SYM2ID(symbol)));
+}
+
+U_EXTERN void Init_u(void);
+void
+Init_u(void)
+{
+        VALUE mU = rb_define_module("U");
+
+        Init_u_buffer(mU);
+        Init_u_string(mU);
+}