character-encodings 0.2.0

data/ext/encoding/character/utf-8/rb_includes.h

@@ -0,0 +1,18 @@
+/*
+ * contents: Standard includes for method definitions.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#ifndef RB_INCLUDES_H
+#define RB_INCLUDES_H
+
+#include <ruby.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "unicode.h"
+#include "private.h"
+#include "rb_methods.h"
+
+#endif /* RB_INCLUDES_H */

data/ext/encoding/character/utf-8/rb_methods.h

@@ -0,0 +1,49 @@
+/*
+ * contents: Method declarations.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#ifndef RB_METHODS_H
+#define RB_METHODS_H
+
+VALUE rb_utf_collate(UNUSED(VALUE self), VALUE str, VALUE other) HIDDEN;
+VALUE rb_utf_downcase(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_length(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_reverse(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_upcase(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_aref_m(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_aset_m(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_casecmp(UNUSED(VALUE self), VALUE str1, VALUE str2) HIDDEN;
+VALUE rb_utf_center(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_ljust(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_rjust(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_chomp_bang(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_chomp(int argc, VALUE *argv, VALUE self) HIDDEN;
+VALUE rb_utf_chop_bang(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_chop(VALUE self, VALUE str) HIDDEN;
+VALUE rb_utf_count(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_delete_bang(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_delete(int argc, VALUE *argv, VALUE self) HIDDEN;
+VALUE rb_utf_each_char(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_index_m(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_insert(UNUSED(VALUE self), VALUE str, VALUE index,
+                    VALUE other) HIDDEN;
+VALUE rb_utf_lstrip_bang(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_lstrip(VALUE self, VALUE str) HIDDEN;
+VALUE rb_utf_rindex_m(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_rstrip_bang(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_rstrip(VALUE self, VALUE str) HIDDEN;
+VALUE rb_utf_squeeze_bang(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_squeeze(int argc, VALUE *argv, VALUE self) HIDDEN;
+VALUE rb_utf_strip_bang(VALUE self, VALUE str) HIDDEN;
+VALUE rb_utf_strip(VALUE self, VALUE str) HIDDEN;
+VALUE rb_utf_to_i(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+VALUE rb_utf_hex(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_oct(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_tr(UNUSED(VALUE self), VALUE str, VALUE from, VALUE to) HIDDEN;
+VALUE rb_utf_tr_s(UNUSED(VALUE self), VALUE str, VALUE from, VALUE to) HIDDEN;
+VALUE rb_utf_foldcase(UNUSED(VALUE self), VALUE str) HIDDEN;
+VALUE rb_utf_normalize(int argc, VALUE *argv, UNUSED(VALUE self)) HIDDEN;
+
+#endif /* RB_METHODS_H */

data/ext/encoding/character/utf-8/rb_utf_aref.c

@@ -0,0 +1,111 @@
+/*
+ * contents: UTF8.aref module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+#include <re.h>
+
+static VALUE
+rb_utf_substr(VALUE str, long offset, long len)
+{
+        if (len < 0)
+                return Qnil;
+
+        char *begin, *limit;
+        if (!rb_utf_begin_from_offset(str, offset, &begin, &limit))
+                return Qnil;
+        char *end = _utf_offset_to_pointer_failable(begin, len, limit);
+        if (end == NULL)
+                end = limit;
+
+        VALUE substr = (begin == end) ?
+                rb_utf_new5(str, NULL, 0) :
+                rb_utf_new5(str, begin, end - begin);
+
+        OBJ_INFECT(substr, str);
+
+        return substr;
+}
+
+static VALUE
+rb_utf_substr_and_infect(VALUE str, long offset, long len, VALUE source)
+{
+        VALUE substr = rb_utf_substr(str, offset, len);
+        OBJ_INFECT(substr, source);
+        return substr;
+}
+
+/* XXX: Stolen straight from string.c. */
+static VALUE
+rb_str_subpat(VALUE str, VALUE re, int nth)
+{
+        if (rb_reg_search(re, str, 0, 0) >= 0)
+                return rb_reg_nth_match(nth, rb_backref_get());
+
+        return Qnil;
+}
+
+static VALUE
+rb_utf_aref_num(VALUE str, long offset)
+{
+        char *begin, *limit;
+        if (!rb_utf_begin_from_offset(str, offset, &begin, &limit))
+                return Qnil;
+
+        char *end = rb_utf_next_validated(begin, limit);
+
+        return rb_utf_new(begin, end - begin);
+}
+
+static VALUE
+rb_utf_aref_default(VALUE str, VALUE index)
+{
+        long n_chars = utf_length_n(RSTRING(str)->ptr, RSTRING(str)->len);
+
+        long begin, len;
+        switch (rb_range_beg_len(index, &begin, &len, n_chars, 0)) {
+        case Qfalse:
+                return rb_utf_aref_num(str, NUM2LONG(index));
+        case Qnil:
+                return Qnil;
+        default:
+                return rb_utf_substr_and_infect(str, begin, len, index);
+        }
+}
+
+static VALUE
+rb_utf_aref(VALUE str, VALUE index)
+{
+        switch (TYPE(index)) {
+        case T_FIXNUM:
+                return rb_utf_aref_num(str, FIX2LONG(index));
+        case T_REGEXP:
+                return rb_str_subpat(str, index, 0);
+        case T_STRING:
+                if (rb_utf_index(str, index, 0) != -1)
+                        return rb_utf_dup(index);
+                return Qnil;
+        default:
+                return rb_utf_aref_default(str, index);
+        }
+}
+
+VALUE
+rb_utf_aref_m(int argc, VALUE *argv, UNUSED(VALUE self))
+{
+        StringValue(argv[0]);
+
+        if (argc > 3 || argc < 2)
+                rb_raise(rb_eArgError,
+                         "wrong number of arguments (%d for 2)", argc);
+
+        if (argc == 2)
+                return rb_utf_aref(argv[0], argv[1]);
+
+        if (TYPE(argv[1]) == T_REGEXP)
+                return rb_str_subpat(argv[0], argv[1], NUM2INT(argv[2]));
+
+        return rb_utf_substr(argv[0], NUM2INT(argv[1]), NUM2INT(argv[2]));
+}

data/ext/encoding/character/utf-8/rb_utf_aset.c

@@ -0,0 +1,105 @@
+/*
+ * contents: UTF8.aset module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+#include <re.h>
+
+/* XXX: Stolen straight from string.c. */
+#define BEG(no) regs->beg[no]
+#define END(no) regs->end[no]
+
+static VALUE
+rb_str_subpat_set(VALUE str, VALUE re, int nth, VALUE val)
+{
+    VALUE match;
+    long start, end, len;
+
+    if (rb_reg_search(re, str, 0, 0) < 0) {
+	rb_raise(rb_eIndexError, "regexp not matched");
+    }
+    match = rb_backref_get();
+    if (nth >= RMATCH(match)->regs->num_regs) {
+      out_of_range:
+	rb_raise(rb_eIndexError, "index %d out of regexp", nth);
+    }
+    if (nth < 0) {
+	if (-nth >= RMATCH(match)->regs->num_regs) {
+	    goto out_of_range;
+	}
+	nth += RMATCH(match)->regs->num_regs;
+    }
+
+    start = RMATCH(match)->BEG(nth);
+    if (start == -1) {
+	rb_raise(rb_eIndexError, "regexp group %d not matched", nth);
+    }
+    end = RMATCH(match)->END(nth);
+    len = end - start;
+    rb_str_update(str, start, len, val);
+
+    return val;
+}
+
+static VALUE
+rb_utf_aset_num(VALUE str, long offset, VALUE replacement)
+{
+        return rb_utf_update(str, offset, 1, replacement);
+}
+
+static VALUE
+rb_utf_aset_default(VALUE str, VALUE index, VALUE replacement)
+{
+        long n_chars = utf_length_n(RSTRING(str)->ptr, RSTRING(str)->len);
+
+        long begin, len;
+        if (rb_range_beg_len(index, &begin, &len, n_chars, 2))
+                return rb_utf_update(str, begin, len, replacement);
+
+        return rb_utf_aset_num(str, NUM2LONG(index), replacement);
+}
+
+static VALUE
+rb_utf_aset(VALUE str, VALUE index, VALUE replacement)
+{
+        switch (TYPE(index)) {
+        case T_FIXNUM:
+                return rb_utf_aset_num(str, FIX2LONG(index), replacement);
+        case T_BIGNUM:
+                return rb_utf_aset_num(str, NUM2LONG(index), replacement);
+        case T_REGEXP:
+                return rb_str_subpat_set(str, index, 0, replacement);
+        case T_STRING: {
+                long begin = rb_utf_index(str, index, 0);
+                if (begin < 0)
+                        rb_raise(rb_eIndexError, "string not matched");
+                return rb_utf_update(str,
+                                     begin,
+                                     utf_length_n(RSTRING(index)->ptr,
+                                                  RSTRING(index)->len),
+                                     replacement);
+        }
+        default:
+                return rb_utf_aset_default(str, index, replacement);
+        }
+}
+
+VALUE
+rb_utf_aset_m(int argc, VALUE *argv, UNUSED(VALUE self))
+{
+        if (argc > 4 || argc < 3)
+                rb_raise(rb_eArgError,
+                         "wrong number of arguments (%d for 3)", argc);
+
+        StringValue(argv[0]);
+
+        if (argc == 3)
+                return rb_utf_aset(argv[0], argv[1], argv[2]);
+
+        if (TYPE(argv[1]) == T_REGEXP)
+                return rb_str_subpat_set(argv[0], argv[1], NUM2INT(argv[2]), argv[3]);
+
+        return rb_utf_update(argv[0], NUM2LONG(argv[1]), NUM2LONG(argv[2]), argv[3]);
+}

data/ext/encoding/character/utf-8/rb_utf_casecmp.c

@@ -0,0 +1,24 @@
+/*
+ * contents: UTF8.casecmp module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+
+VALUE
+rb_utf_casecmp(UNUSED(VALUE self), VALUE str1, VALUE str2)
+{
+        StringValue(str1);
+        StringValue(str2);
+
+        char *folded1 = utf_foldcase(RSTRING(str1)->ptr);
+        char *folded2 = utf_foldcase(RSTRING(str2)->ptr);
+
+        int result = utf_collate(folded1, folded2);
+
+        free(folded2);
+        free(folded1);
+
+        return INT2FIX(result);
+}

data/ext/encoding/character/utf-8/rb_utf_chomp.c

@@ -0,0 +1,114 @@
+/*
+ * contents: UTF8.chomp module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+
+static VALUE
+rb_utf_chomp_default(VALUE str)
+{
+        rb_str_modify(str);
+
+        const char *end = RSTRING(str)->ptr + RSTRING(str)->len;
+
+        char *last = utf_find_prev(RSTRING(str)->ptr, end);
+        if (last == NULL)
+                return Qnil;
+
+        if (_utf_char_validated(last, end) == '\n') {
+                char *last_but_one = utf_find_prev(RSTRING(str)->ptr, last);
+
+                if (last_but_one != NULL && utf_char(last_but_one) == '\r')
+                        last = last_but_one;
+        } else if (!unichar_isnewline(utf_char(last))) {
+                return Qnil;
+        }
+
+        RSTRING(str)->len -= (RSTRING(str)->ptr + RSTRING(str)->len) - last;
+        *last = '\0';
+
+        return str;
+}
+
+static VALUE
+rb_utf_chomp_newlines(VALUE str)
+{
+        char *begin = RSTRING(str)->ptr;
+        char *end = begin + RSTRING(str)->len;
+
+        char *last = end;
+        while (last > begin) {
+                char *last_but_one = utf_find_prev(begin, last);
+                if (last == NULL || !unichar_isnewline(utf_char(last_but_one)))
+                        break;
+                last = last_but_one;
+        }
+
+        if (last == end)
+                return Qnil;
+
+        rb_str_modify(str);
+        RSTRING(str)->len -= end - last;
+        *last = '\0';
+
+        return str;
+}
+
+VALUE
+rb_utf_chomp_bang(int argc, VALUE *argv, UNUSED(VALUE self))
+{
+        VALUE str, rs;
+
+        rb_scan_args(argc, argv, "11", &str, &rs);
+
+        if (RSTRING(str)->len == 0)
+                return Qnil;
+
+        if (argc == 1) {
+                rs = rb_rs;
+                if (rs == rb_default_rs)
+                        rb_utf_chomp_default(str);
+        }
+
+        if (NIL_P(rs))
+                return Qnil;
+
+        StringValue(rs);
+
+        long rs_len = RSTRING(rs)->len;
+        if (rs_len == 0)
+                return rb_utf_chomp_newlines(str);
+
+        long len = RSTRING(str)->len;
+        if (rs_len > len)
+                return Qnil;
+
+        char last_char = RSTRING(rs)->ptr[rs_len - 1];
+        if (rs_len == 1 && last_char == '\n')
+                rb_utf_chomp_default(str);
+
+        char *p = RSTRING(str)->ptr;
+
+        if (p[len - 1] != last_char ||
+            (rs_len > 1 &&
+             rb_memcmp(RSTRING(rs)->ptr, p + len - rs_len, rs_len) != 0))
+                return Qnil;
+
+        rb_str_modify(str);
+        RSTRING(str)->len -= rs_len;
+        RSTRING(str)->ptr[RSTRING(str)->len] = '\0';
+
+        return str;
+}
+
+VALUE
+rb_utf_chomp(int argc, VALUE *argv, VALUE self)
+{
+        StringValue(argv[0]);
+        argv[0] = rb_utf_dup(argv[0]);
+        rb_utf_chomp_bang(argc, argv, self);
+        return argv[0];
+}
+

data/ext/encoding/character/utf-8/rb_utf_chop.c

@@ -0,0 +1,44 @@
+/*
+ * contents: UTF8.chop module function.
+ *
+ * Copyright © 2006 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include "rb_includes.h"
+
+VALUE
+rb_utf_chop_bang(UNUSED(VALUE self), VALUE str)
+{
+        StringValue(str);
+
+        if (RSTRING(str)->len == 0)
+                return Qnil;
+
+        rb_str_modify(str);
+
+        const char *end = RSTRING(str)->ptr + RSTRING(str)->len;
+
+        char *last = rb_utf_prev_validated(RSTRING(str)->ptr, end);
+
+        if (_utf_char_validated(last, end) == '\n') {
+                char *last_but_one = utf_find_prev(RSTRING(str)->ptr, last);
+
+                if (last_but_one != NULL && utf_char(last_but_one) == '\r')
+                        last = last_but_one;
+        } else if (!unichar_isnewline(utf_char(last))) {
+                return Qnil;
+        }
+
+        RSTRING(str)->len -= (RSTRING(str)->ptr + RSTRING(str)->len) - last;
+        *last = '\0';
+
+        return str;
+}
+
+VALUE
+rb_utf_chop(VALUE self, VALUE str)
+{
+        str = rb_utf_dup(str);
+        rb_utf_chop_bang(self, str);
+        return str;
+}