u 0.5.0 → 1.0.0

data/ext/encoding/character/utf-8/rb_utf_rstrip.c

@@ -1,51 +0,0 @@
-/*
- * contents: UTF8.rstrip module function.
- *
- * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
- */
-
-#include "rb_includes.h"
-
-VALUE
-rb_utf_rstrip_bang(UNUSED(VALUE self), VALUE str)
-{
-        StringValue(str);
-        const char *begin = RSTRING(str)->ptr;
-        if (begin == NULL || RSTRING(str)->len == 0)
-                return Qnil;
-
-        const char *end = begin + RSTRING(str)->len;
-        const char *t = end;
-
-        /* Remove trailing '\0'’s. */
-        while (t > begin && t[-1] == '\0')
-                t--;
-
-        /* Remove trailing spaces. */
-        while (t > begin) {
-                /* FIXME: Should we be validating here? */
-                const char *prev = rb_utf_prev_validated(begin, t);
-                
-                if (!unichar_isspace(utf_char(prev)))
-                        break;
-
-                t = prev;
-        }
-
-        if (t == end)
-                return Qnil;
-
-        rb_str_modify(str);
-        RSTRING(str)->len = t - begin;
-        RSTRING(str)->ptr[RSTRING(str)->len] = '\0';
-
-        return str;
-}
-
-VALUE
-rb_utf_rstrip(VALUE self, VALUE str)
-{
-        str = rb_utf_dup(str);
-        rb_utf_rstrip_bang(self, str);
-        return str;
-}

data/ext/encoding/character/utf-8/rb_utf_squeeze.c

@@ -1,70 +0,0 @@
-/*
- * contents: UTF8.squeeze module function.
- *
- * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
- */
-
-#include "rb_includes.h"
-#include "rb_utf_internal_tr.h"
-
-VALUE
-rb_utf_squeeze_bang(int argc, VALUE *argv, UNUSED(VALUE self))
-{
-        need_at_least_n_arguments(argc, 1);
-
-        VALUE str = argv[0];
-        StringValue(str);
-        if (RSTRING(str)->len == 0)
-                return Qnil;
-
-        unsigned int table[TR_TABLE_SIZE];
-        if (argc == 1)
-                for (int i = 0; i < TR_TABLE_SIZE; i++)
-                        table[i] = ~0U;
-        else
-                tr_setup_table_from_strings(table, argc - 1, &argv[1]);
-
-        rb_str_modify(str);
-
-        char *begin = RSTRING(str)->ptr;
-        char const *end = begin + RSTRING(str)->len;
-
-        /* We know that there is a character to eat (if the input isn’t
-         * invalid), as we’ve already verified that RSTRING(str)->len > 0, so
-         * ‘s_end’ must lay beyond ‘s’.  Also, as we validate when we fetch the
-         * character, there’s no need to validate the call to utf_next(). */
-        unichar previous = _utf_char_validated(begin, end);
-        char *s = utf_next(begin);
-        char *t = s;
-        while (s < end) {
-                unichar c = _utf_char_validated(s, end);
-                char *next = utf_next(s);
-
-                if (c != previous || !tr_table_lookup(table, c)) {
-                        memmove(t, s, next - s);
-                        t += next - s;
-                        previous = c;
-                }
-
-                s = next;
-        }
-        *t = '\0';
-
-        if (t - begin != RSTRING(str)->len) {
-                RSTRING(str)->len = t - begin;
-                return str;
-        }
-
-        return Qnil;
-}
-
-VALUE
-rb_utf_squeeze(int argc, VALUE *argv, VALUE self)
-{
-        need_at_least_n_arguments(argc, 1);
-
-        StringValue(argv[0]);
-        argv[0] = rb_utf_dup(argv[0]);
-        rb_utf_squeeze_bang(argc, argv, self);
-        return argv[0];
-}

data/ext/encoding/character/utf-8/rb_utf_strip.c

@@ -1,27 +0,0 @@
-/*
- * contents: UTF8.strip module function.
- *
- * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
- */
-
-#include "rb_includes.h"
-
-VALUE
-rb_utf_strip_bang(VALUE self, VALUE str)
-{
-        VALUE left = rb_utf_lstrip_bang(self, str);
-        VALUE right = rb_utf_rstrip_bang(self, str);
-
-        if (NIL_P(left) && NIL_P(right))
-                return Qnil;
-
-        return str;
-}
-
-VALUE
-rb_utf_strip(VALUE self, VALUE str)
-{
-        str = rb_utf_dup(str);
-        rb_utf_strip_bang(self, str);
-        return str;
-}

data/ext/encoding/character/utf-8/rb_utf_to_i.c

@@ -1,25 +0,0 @@
-/*
- * contents: UTF8.to_i module function.
- *
- * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
- */
-
-#include "rb_includes.h"
-#include "rb_utf_internal_bignum.h"
-
-VALUE
-rb_utf_to_i(int argc, VALUE *argv, UNUSED(VALUE self))
-{
-        VALUE str, rbbase;
-
-        int base = 10;
-        if (rb_scan_args(argc, argv, "11", &str, &rbbase) == 2)
-                base = NUM2INT(rbbase);
-
-        /* XXX: this test is actually unnecessary, as this will be checked in
-         * rb_utf_to_inum() as well. */
-        if (base < 0)
-                rb_raise(rb_eArgError, "illegal radix %d", base);
-
-        return rb_utf_to_inum(str, base, false);
-}

data/ext/encoding/character/utf-8/rb_utf_tr.c

@@ -1,250 +0,0 @@
-/*
- * contents: UTF8.tr module function.
- *
- * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
- */
-
-#include "rb_includes.h"
-#include "rb_utf_internal_tr.h"
-
-struct tr_range
-{
-        unichar begin;
-        unichar end;
-};
-
-static int
-tr_ranges_setup(struct tr *tr, struct tr_range *ranges)
-{
-        int n = 0;
-        bool was_inside_range = false;
-        while (tr_next(tr) != TR_FINISHED) {
-                if (tr->inside_range) {
-                        if (!was_inside_range) {
-                                ranges[n].begin = tr->now;
-                                was_inside_range = true;
-                        }
-                } else {
-                        if (was_inside_range)
-                                ranges[n].end = tr->now;
-                        else
-                                ranges[n].begin = ranges[n].end = tr->now;
-                        n++;
-                        was_inside_range = false;
-                }
-        }
-
-        return n;
-}
-
-
-struct tr_trans_closure
-{
-        struct tr_range *from;
-        int n_from;
-        struct tr_range *to;
-        int n_to;
-};
-
-static unichar
-tr_trans_replace_exclude(UNUSED(unichar c), void *closure)
-{
-        return *((unichar *)closure);
-}
-
-static int
-tr_trans_replace_include_offset_of(struct tr_range *ranges, int range)
-{
-        int offset = 0;
-
-        for (int i = 0; i < range; i++)
-                offset += ranges[i].end - ranges[i].begin + 1;
-
-        return offset;
-}
-
-static unichar
-tr_trans_replace_include(unichar c, void *v_closure)
-{
-        struct tr_trans_closure *closure = (struct tr_trans_closure *)v_closure;
-
-        for (int i = closure->n_from - 1; i >= 0; i--) {
-                if (closure->from[i].begin >= c && closure->from[i].end <= c) {
-                        int offset = tr_trans_replace_include_offset_of(closure->from, i);
-                        int j;
-                        for (j = 0; j < closure->n_to && offset > 0; j++)
-                                offset -= closure->to[j].end - closure->to[j].begin + 1;
-
-                        if (offset > 0)
-                                return closure->to[closure->n_to - 1].end;
-
-                        return closure->to[j].end - offset;
-                }
-        }
-
-        return closure->to[closure->n_to - 1].end;
-}
-
-static VALUE
-tr_trans_do(VALUE src, unsigned int *translation,
-            unichar (*replace)(unichar, void *), void *closure, bool squeeze,
-            UNUSED(bool replace_content))
-{
-        VALUE dst = Qnil;
-        long len;
-
-again:
-        len = 0;
-
-        const char *s = RSTRING(src)->ptr;
-        const char *s_end = s + RSTRING(src)->len;
-
-        char *t = NULL;
-        
-        if (dst != Qnil)
-                t = RSTRING(dst)->ptr;
-
-        bool modified = false;
-
-        /* TODO: this should really be refactored… */
-        if (squeeze) {
-                unichar prev_c = -1;
-
-                while (s < s_end) {
-                        unichar c0 = utf_char(s);
-
-                        const char *prev = s;
-                        s = utf_next(s);
-
-                        if (tr_table_lookup(translation, c0)) {
-                                unichar c = replace(c0, closure);
-                                if (prev_c == c)
-                                        continue;
-                                prev_c = c;
-                                len += unichar_to_utf(c, (t != NULL) ? t + len : NULL);
-                                modified = true;
-                        } else {
-                                prev_c = -1;
-                                if (t != NULL)
-                                        memcpy(t + len, prev, s - prev);
-                                len += s - prev;
-                        }
-
-                }
-
-                if (RSTRING(src)->len > (t + len - RSTRING(src)->ptr))
-                        modified = true;
-        } else {
-                while (s < s_end) {
-                        unichar c = utf_char(s);
-
-                        const char *prev = s;
-                        s = utf_next(s);
-
-                        if (tr_table_lookup(translation, c)) {
-                                len += unichar_to_utf(replace(c, closure),
-                                                      (t != NULL) ? t + len : NULL);
-                                modified = true;
-                        } else {
-                                if (t != NULL)
-                                        memcpy(t + len, prev, s - prev);
-                                len += s - prev;
-                        }
-                }
-        }
-
-#ifdef RB_STR_REPLACE_IS_EXTERN
-        if (replace_content && !modified)
-                return Qnil;
-#endif
-
-        if (dst == Qnil) {
-#ifdef RB_STR_REPLACE_IS_EXTERN
-                if (replace_content && len <= RSTRING(src)->len)
-                        dst = src;
-                else
-#endif
-                        dst = rb_str_buf_new(len);
-                goto again;
-        }
-
-        t[len] = '\0';
-        RSTRING(dst)->len = len;
-
-#ifdef RB_STR_REPLACE_IS_EXTERN
-        if (dst != src && replace_content) {
-                rb_str_replace(src, dst);
-                return src;
-        }
-#endif
-
-        return dst;
-}
-
-static VALUE
-tr_trans(VALUE str, VALUE from, VALUE to, bool squeeze, bool replace_content)
-{
-        StringValue(str);
-        StringValue(from);
-        StringValue(to);
-
-        if (RSTRING(str)->ptr == NULL || RSTRING(str)->len == 0)
-                return replace_content ? Qnil : str;
-
-        if (RSTRING(to)->len == 0)
-                return rb_utf_delete_bang(1, &from, str);
-
-        struct tr tr_from;
-        tr_init(&tr_from,
-                RSTRING(from)->ptr,
-                RSTRING(from)->ptr + RSTRING(from)->len);
-
-        struct tr tr_to;
-        tr_init(&tr_to,
-                RSTRING(to)->ptr,
-                RSTRING(to)->ptr + RSTRING(to)->len);
-
-        unsigned int translation[TR_TABLE_SIZE];
-        tr_setup_table(from, translation, true);
-
-        tr_init(&tr_from,
-                RSTRING(from)->ptr,
-                RSTRING(from)->ptr + RSTRING(from)->len);
-        if (tr_should_exclude(&tr_from)) {
-                /* This case is easy.  Just include everything by default and
-                 * exclude the rest as always.  Replace characters found by the
-                 * last character found in tr_to. */
-                while (tr_next(&tr_to) != TR_FINISHED)
-                       ; /* We just need the last replacement character. */
-                return tr_trans_do(str, translation, tr_trans_replace_exclude,
-                                   &tr_to.now, squeeze, replace_content);
-        } else {
-                /* This case is hard.  We need a full-fledged lookup of what
-                 * character to translate to, not simply a check whether to
-                 * include it or not. */
-                struct tr_trans_closure trans_closure;
-
-                struct tr_range from_ranges[utf_length_n(RSTRING(from)->ptr, RSTRING(from)->len)];
-                trans_closure.from = from_ranges;
-                trans_closure.n_from = tr_ranges_setup(&tr_from, from_ranges);
-
-                struct tr_range to_ranges[utf_length_n(RSTRING(to)->ptr, RSTRING(to)->len)];
-                trans_closure.to = to_ranges;
-                trans_closure.n_to = tr_ranges_setup(&tr_to, to_ranges);
-
-                return tr_trans_do(str, translation, tr_trans_replace_include,
-                                   &trans_closure, squeeze, replace_content);
-        }
-}
-
-VALUE
-rb_utf_tr(UNUSED(VALUE self), VALUE str, VALUE from, VALUE to)
-{
-        return tr_trans(str, from, to, false, false);
-}
-
-VALUE
-rb_utf_tr_s(UNUSED(VALUE self), VALUE str, VALUE from, VALUE to)
-{
-        return tr_trans(str, from, to, true, false);
-}

data/ext/encoding/character/utf-8/rb_utf_upcase.c

@@ -1,13 +0,0 @@
-/*
- * contents: UTF8.upcase module function.
- *
- * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
- */
-
-#include "rb_includes.h"
-
-VALUE
-rb_utf_upcase(UNUSED(VALUE self), VALUE str)
-{
-        return rb_utf_alloc_using(utf_upcase(StringValuePtr(str)));
-}

data/ext/encoding/character/utf-8/tables.h

@@ -1,38 +0,0 @@
-/*
- * contents: Functions for dealing with Unicode tables.
- *
- * Copyright © 2007 Nikolai Weibull <now@bitwi.se>
- */
-
-#ifndef TABLES_H
-#define TABLES_H
-
-
-/*
-static inline int
-split_unicode_table_lookup_page(const uint8_t data[][256], int16_t page, unichar c)
-{
-        return (page >= UNICODE_MAX_TABLE_INDEX) ?
-                page - UNICODE_MAX_TABLE_INDEX :
-                data[page][c & 0xff];
-}
-
-static inline int
-split_unicode_table_lookup(const uint8_t data[][256], const int16_t part1[], const int16_t part2[], unichar c, int fallback)
-{
-	if (c <= UNICODE_LAST_CHAR_PART1)
-                return split_unicode_table_lookup_page(data,
-                                                       part1[c >> 8],
-                                                       c);
-
-        if (c >= UNICODE_FIRST_CHAR_PART2 && c <= UNICODE_LAST_CHAR)
-                return split_unicode_table_lookup_page(data,
-                                                       part2[(c - UNICODE_FIRST_CHAR_PART2) >> 8],
-                                                       c);
-
-        return fallback;
-}
-*/
-
-
-#endif /* TABLES_H */