utf8 0.1.3 → 0.1.4

data/ext/utf8/string_utf8.c

@@ -3,6 +3,8 @@
 
 extern VALUE intern_as_utf8;
 
+#define REPLACEMENT_CHAR '?'
+
 /*
  * Document-class: String::UTF-8
  */
@@ -112,7 +114,7 @@ static VALUE rb_cString_UTF8_valid(int argc, VALUE *argv, VALUE self) {
   }
 
   for(; i<len; i+=lastCharLen) {
-    lastCharLen = utf8CharLen(str, len);
+    lastCharLen = utf8CharLen(str+i, len);
     if (lastCharLen < 0) {
       return Qfalse;
     }
@@ -327,6 +329,42 @@ static VALUE rb_cString_UTF8_slice(int argc, VALUE *argv, VALUE self) {
   }
 }
 
+/*
+ * call-seq: clean
+ *
+ * Iterates over the string, replacing invalid UTF-8 characters with '?'
+ *
+ * Returns: a new String
+ */
+static VALUE rb_cString_UTF8_clean(VALUE self) {
+  unsigned char *str;
+  unsigned char *out;
+  unsigned char replace;
+  size_t len;
+  int8_t curCharLen;
+  size_t i;
+  VALUE rb_out;
+
+  str = (unsigned char *)RSTRING_PTR(self);
+  len = RSTRING_LEN(self);
+  replace = REPLACEMENT_CHAR;
+  out = xmalloc(len);
+
+  for(i=0; i<len; i++) {
+    curCharLen = utf8CharLen(str+i, len);
+    if (curCharLen < 0) {
+      *(out+i) = replace;
+    } else {
+      *(out+i) = *(str+i);
+    }
+  }
+
+  rb_out = rb_str_new((const char*)out, len);
+  AS_UTF8(rb_out);
+
+  return rb_out;
+}
+
 void init_String_UTF8() {
   VALUE rb_cString_UTF8 = rb_define_class_under(rb_cString, "UTF8", rb_cString);
 
@@ -335,4 +373,5 @@ void init_String_UTF8() {
   rb_define_method(rb_cString_UTF8, "[]",        rb_cString_UTF8_slice, -1);
   rb_define_method(rb_cString_UTF8, "each_codepoint", rb_cString_UTF8_each_codepoint, -1);
   rb_define_method(rb_cString_UTF8, "valid?", rb_cString_UTF8_valid, -1);
+  rb_define_method(rb_cString_UTF8, "clean", rb_cString_UTF8_clean, 0);
 }

data/lib/utf8/version.rb

@@ -1,5 +1,5 @@
 class String
   class UTF8 < ::String
-    VERSION = "0.1.3"
+    VERSION = "0.1.4"
   end
 end

data/spec/string_spec.rb

@@ -50,6 +50,12 @@ describe String::UTF8 do
     @utf8.chars.to_a[0].class.should eql(String::UTF8)
   end
 
+  it "clean should replace invalid utf8 chars with '?'" do
+    orig = "provided by Cristian Rodr\355guez."
+    clean = "provided by Cristian Rodr?guez."
+    orig.as_utf8.clean.should eql(clean)
+  end
+
   context "#length and #size" do
     it "should be utf8-aware" do
       @utf8.length.should eql(@utf8_len)
@@ -185,6 +191,8 @@ describe String::UTF8 do
 
       utf8.valid?.should be_false
       @utf8.valid?.should be_true
+
+      "provided by Cristian Rodr\355guez.".as_utf8.should_not be_valid
     end
 
     it "should test validity using a maximum codepoint" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: utf8
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 19
   prerelease: 
   segments: 
   - 0
   - 1
-  - 3
-  version: 0.1.3
+  - 4
+  version: 0.1.4
 platform: ruby
 authors: 
 - Brian Lopez
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-02 00:00:00 -07:00
+date: 2011-07-21 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -129,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.3.10
 signing_key: 
 specification_version: 3
 summary: A lightweight UTF8-aware String class meant for use with Ruby 1.8