u 0.5.0

data/ext/encoding/character/utf-8/rb_utf_rstrip.c

@@ -0,0 +1,51 @@
+/*
+ * contents: UTF8.rstrip module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+
+VALUE
+rb_utf_rstrip_bang(UNUSED(VALUE self), VALUE str)
+{
+        StringValue(str);
+        const char *begin = RSTRING(str)->ptr;
+        if (begin == NULL || RSTRING(str)->len == 0)
+                return Qnil;
+
+        const char *end = begin + RSTRING(str)->len;
+        const char *t = end;
+
+        /* Remove trailing '\0'’s. */
+        while (t > begin && t[-1] == '\0')
+                t--;
+
+        /* Remove trailing spaces. */
+        while (t > begin) {
+                /* FIXME: Should we be validating here? */
+                const char *prev = rb_utf_prev_validated(begin, t);
+                
+                if (!unichar_isspace(utf_char(prev)))
+                        break;
+
+                t = prev;
+        }
+
+        if (t == end)
+                return Qnil;
+
+        rb_str_modify(str);
+        RSTRING(str)->len = t - begin;
+        RSTRING(str)->ptr[RSTRING(str)->len] = '\0';
+
+        return str;
+}
+
+VALUE
+rb_utf_rstrip(VALUE self, VALUE str)
+{
+        str = rb_utf_dup(str);
+        rb_utf_rstrip_bang(self, str);
+        return str;
+}

data/ext/encoding/character/utf-8/rb_utf_squeeze.c

@@ -0,0 +1,70 @@
+/*
+ * contents: UTF8.squeeze module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+#include "rb_utf_internal_tr.h"
+
+VALUE
+rb_utf_squeeze_bang(int argc, VALUE *argv, UNUSED(VALUE self))
+{
+        need_at_least_n_arguments(argc, 1);
+
+        VALUE str = argv[0];
+        StringValue(str);
+        if (RSTRING(str)->len == 0)
+                return Qnil;
+
+        unsigned int table[TR_TABLE_SIZE];
+        if (argc == 1)
+                for (int i = 0; i < TR_TABLE_SIZE; i++)
+                        table[i] = ~0U;
+        else
+                tr_setup_table_from_strings(table, argc - 1, &argv[1]);
+
+        rb_str_modify(str);
+
+        char *begin = RSTRING(str)->ptr;
+        char const *end = begin + RSTRING(str)->len;
+
+        /* We know that there is a character to eat (if the input isn’t
+         * invalid), as we’ve already verified that RSTRING(str)->len > 0, so
+         * ‘s_end’ must lay beyond ‘s’.  Also, as we validate when we fetch the
+         * character, there’s no need to validate the call to utf_next(). */
+        unichar previous = _utf_char_validated(begin, end);
+        char *s = utf_next(begin);
+        char *t = s;
+        while (s < end) {
+                unichar c = _utf_char_validated(s, end);
+                char *next = utf_next(s);
+
+                if (c != previous || !tr_table_lookup(table, c)) {
+                        memmove(t, s, next - s);
+                        t += next - s;
+                        previous = c;
+                }
+
+                s = next;
+        }
+        *t = '\0';
+
+        if (t - begin != RSTRING(str)->len) {
+                RSTRING(str)->len = t - begin;
+                return str;
+        }
+
+        return Qnil;
+}
+
+VALUE
+rb_utf_squeeze(int argc, VALUE *argv, VALUE self)
+{
+        need_at_least_n_arguments(argc, 1);
+
+        StringValue(argv[0]);
+        argv[0] = rb_utf_dup(argv[0]);
+        rb_utf_squeeze_bang(argc, argv, self);
+        return argv[0];
+}

data/ext/encoding/character/utf-8/rb_utf_strip.c

@@ -0,0 +1,27 @@
+/*
+ * contents: UTF8.strip module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+
+VALUE
+rb_utf_strip_bang(VALUE self, VALUE str)
+{
+        VALUE left = rb_utf_lstrip_bang(self, str);
+        VALUE right = rb_utf_rstrip_bang(self, str);
+
+        if (NIL_P(left) && NIL_P(right))
+                return Qnil;
+
+        return str;
+}
+
+VALUE
+rb_utf_strip(VALUE self, VALUE str)
+{
+        str = rb_utf_dup(str);
+        rb_utf_strip_bang(self, str);
+        return str;
+}

data/ext/encoding/character/utf-8/rb_utf_to_i.c

@@ -0,0 +1,25 @@
+/*
+ * contents: UTF8.to_i module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+#include "rb_utf_internal_bignum.h"
+
+VALUE
+rb_utf_to_i(int argc, VALUE *argv, UNUSED(VALUE self))
+{
+        VALUE str, rbbase;
+
+        int base = 10;
+        if (rb_scan_args(argc, argv, "11", &str, &rbbase) == 2)
+                base = NUM2INT(rbbase);
+
+        /* XXX: this test is actually unnecessary, as this will be checked in
+         * rb_utf_to_inum() as well. */
+        if (base < 0)
+                rb_raise(rb_eArgError, "illegal radix %d", base);
+
+        return rb_utf_to_inum(str, base, false);
+}

data/ext/encoding/character/utf-8/rb_utf_tr.c

@@ -0,0 +1,250 @@
+/*
+ * contents: UTF8.tr module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+#include "rb_utf_internal_tr.h"
+
+struct tr_range
+{
+        unichar begin;
+        unichar end;
+};
+
+static int
+tr_ranges_setup(struct tr *tr, struct tr_range *ranges)
+{
+        int n = 0;
+        bool was_inside_range = false;
+        while (tr_next(tr) != TR_FINISHED) {
+                if (tr->inside_range) {
+                        if (!was_inside_range) {
+                                ranges[n].begin = tr->now;
+                                was_inside_range = true;
+                        }
+                } else {
+                        if (was_inside_range)
+                                ranges[n].end = tr->now;
+                        else
+                                ranges[n].begin = ranges[n].end = tr->now;
+                        n++;
+                        was_inside_range = false;
+                }
+        }
+
+        return n;
+}
+
+
+struct tr_trans_closure
+{
+        struct tr_range *from;
+        int n_from;
+        struct tr_range *to;
+        int n_to;
+};
+
+static unichar
+tr_trans_replace_exclude(UNUSED(unichar c), void *closure)
+{
+        return *((unichar *)closure);
+}
+
+static int
+tr_trans_replace_include_offset_of(struct tr_range *ranges, int range)
+{
+        int offset = 0;
+
+        for (int i = 0; i < range; i++)
+                offset += ranges[i].end - ranges[i].begin + 1;
+
+        return offset;
+}
+
+static unichar
+tr_trans_replace_include(unichar c, void *v_closure)
+{
+        struct tr_trans_closure *closure = (struct tr_trans_closure *)v_closure;
+
+        for (int i = closure->n_from - 1; i >= 0; i--) {
+                if (closure->from[i].begin >= c && closure->from[i].end <= c) {
+                        int offset = tr_trans_replace_include_offset_of(closure->from, i);
+                        int j;
+                        for (j = 0; j < closure->n_to && offset > 0; j++)
+                                offset -= closure->to[j].end - closure->to[j].begin + 1;
+
+                        if (offset > 0)
+                                return closure->to[closure->n_to - 1].end;
+
+                        return closure->to[j].end - offset;
+                }
+        }
+
+        return closure->to[closure->n_to - 1].end;
+}
+
+static VALUE
+tr_trans_do(VALUE src, unsigned int *translation,
+            unichar (*replace)(unichar, void *), void *closure, bool squeeze,
+            UNUSED(bool replace_content))
+{
+        VALUE dst = Qnil;
+        long len;
+
+again:
+        len = 0;
+
+        const char *s = RSTRING(src)->ptr;
+        const char *s_end = s + RSTRING(src)->len;
+
+        char *t = NULL;
+        
+        if (dst != Qnil)
+                t = RSTRING(dst)->ptr;
+
+        bool modified = false;
+
+        /* TODO: this should really be refactored… */
+        if (squeeze) {
+                unichar prev_c = -1;
+
+                while (s < s_end) {
+                        unichar c0 = utf_char(s);
+
+                        const char *prev = s;
+                        s = utf_next(s);
+
+                        if (tr_table_lookup(translation, c0)) {
+                                unichar c = replace(c0, closure);
+                                if (prev_c == c)
+                                        continue;
+                                prev_c = c;
+                                len += unichar_to_utf(c, (t != NULL) ? t + len : NULL);
+                                modified = true;
+                        } else {
+                                prev_c = -1;
+                                if (t != NULL)
+                                        memcpy(t + len, prev, s - prev);
+                                len += s - prev;
+                        }
+
+                }
+
+                if (RSTRING(src)->len > (t + len - RSTRING(src)->ptr))
+                        modified = true;
+        } else {
+                while (s < s_end) {
+                        unichar c = utf_char(s);
+
+                        const char *prev = s;
+                        s = utf_next(s);
+
+                        if (tr_table_lookup(translation, c)) {
+                                len += unichar_to_utf(replace(c, closure),
+                                                      (t != NULL) ? t + len : NULL);
+                                modified = true;
+                        } else {
+                                if (t != NULL)
+                                        memcpy(t + len, prev, s - prev);
+                                len += s - prev;
+                        }
+                }
+        }
+
+#ifdef RB_STR_REPLACE_IS_EXTERN
+        if (replace_content && !modified)
+                return Qnil;
+#endif
+
+        if (dst == Qnil) {
+#ifdef RB_STR_REPLACE_IS_EXTERN
+                if (replace_content && len <= RSTRING(src)->len)
+                        dst = src;
+                else
+#endif
+                        dst = rb_str_buf_new(len);
+                goto again;
+        }
+
+        t[len] = '\0';
+        RSTRING(dst)->len = len;
+
+#ifdef RB_STR_REPLACE_IS_EXTERN
+        if (dst != src && replace_content) {
+                rb_str_replace(src, dst);
+                return src;
+        }
+#endif
+
+        return dst;
+}
+
+static VALUE
+tr_trans(VALUE str, VALUE from, VALUE to, bool squeeze, bool replace_content)
+{
+        StringValue(str);
+        StringValue(from);
+        StringValue(to);
+
+        if (RSTRING(str)->ptr == NULL || RSTRING(str)->len == 0)
+                return replace_content ? Qnil : str;
+
+        if (RSTRING(to)->len == 0)
+                return rb_utf_delete_bang(1, &from, str);
+
+        struct tr tr_from;
+        tr_init(&tr_from,
+                RSTRING(from)->ptr,
+                RSTRING(from)->ptr + RSTRING(from)->len);
+
+        struct tr tr_to;
+        tr_init(&tr_to,
+                RSTRING(to)->ptr,
+                RSTRING(to)->ptr + RSTRING(to)->len);
+
+        unsigned int translation[TR_TABLE_SIZE];
+        tr_setup_table(from, translation, true);
+
+        tr_init(&tr_from,
+                RSTRING(from)->ptr,
+                RSTRING(from)->ptr + RSTRING(from)->len);
+        if (tr_should_exclude(&tr_from)) {
+                /* This case is easy.  Just include everything by default and
+                 * exclude the rest as always.  Replace characters found by the
+                 * last character found in tr_to. */
+                while (tr_next(&tr_to) != TR_FINISHED)
+                       ; /* We just need the last replacement character. */
+                return tr_trans_do(str, translation, tr_trans_replace_exclude,
+                                   &tr_to.now, squeeze, replace_content);
+        } else {
+                /* This case is hard.  We need a full-fledged lookup of what
+                 * character to translate to, not simply a check whether to
+                 * include it or not. */
+                struct tr_trans_closure trans_closure;
+
+                struct tr_range from_ranges[utf_length_n(RSTRING(from)->ptr, RSTRING(from)->len)];
+                trans_closure.from = from_ranges;
+                trans_closure.n_from = tr_ranges_setup(&tr_from, from_ranges);
+
+                struct tr_range to_ranges[utf_length_n(RSTRING(to)->ptr, RSTRING(to)->len)];
+                trans_closure.to = to_ranges;
+                trans_closure.n_to = tr_ranges_setup(&tr_to, to_ranges);
+
+                return tr_trans_do(str, translation, tr_trans_replace_include,
+                                   &trans_closure, squeeze, replace_content);
+        }
+}
+
+VALUE
+rb_utf_tr(UNUSED(VALUE self), VALUE str, VALUE from, VALUE to)
+{
+        return tr_trans(str, from, to, false, false);
+}
+
+VALUE
+rb_utf_tr_s(UNUSED(VALUE self), VALUE str, VALUE from, VALUE to)
+{
+        return tr_trans(str, from, to, true, false);
+}

data/ext/encoding/character/utf-8/rb_utf_upcase.c

@@ -0,0 +1,13 @@
+/*
+ * contents: UTF8.upcase module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+
+VALUE
+rb_utf_upcase(UNUSED(VALUE self), VALUE str)
+{
+        return rb_utf_alloc_using(utf_upcase(StringValuePtr(str)));
+}

data/ext/encoding/character/utf-8/tables.h

@@ -0,0 +1,38 @@
+/*
+ * contents: Functions for dealing with Unicode tables.
+ *
+ * Copyright © 2007 Nikolai Weibull <now@bitwi.se>
+ */
+
+#ifndef TABLES_H
+#define TABLES_H
+
+
+/*
+static inline int
+split_unicode_table_lookup_page(const uint8_t data[][256], int16_t page, unichar c)
+{
+        return (page >= UNICODE_MAX_TABLE_INDEX) ?
+                page - UNICODE_MAX_TABLE_INDEX :
+                data[page][c & 0xff];
+}
+
+static inline int
+split_unicode_table_lookup(const uint8_t data[][256], const int16_t part1[], const int16_t part2[], unichar c, int fallback)
+{
+	if (c <= UNICODE_LAST_CHAR_PART1)
+                return split_unicode_table_lookup_page(data,
+                                                       part1[c >> 8],
+                                                       c);
+
+        if (c >= UNICODE_FIRST_CHAR_PART2 && c <= UNICODE_LAST_CHAR)
+                return split_unicode_table_lookup_page(data,
+                                                       part2[(c - UNICODE_FIRST_CHAR_PART2) >> 8],
+                                                       c);
+
+        return fallback;
+}
+*/
+
+
+#endif /* TABLES_H */