charlock_holmes 0.4.1 → 0.5.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    charlock_holmes (0.4.0)
+    charlock_holmes (0.5.0)
 
 GEM
   remote: http://rubygems.org/

data/ext/charlock_holmes/common.h

@@ -0,0 +1,41 @@
+#ifndef CHARLOCK_COMMON_H
+#define CHARLOCK_COMMON_H
+
+// tell rbx not to use it's caching compat layer
+// by doing this we're making a promize to RBX that
+// we'll never modify the pointers we get back from RSTRING_PTR
+#define RSTRING_NOT_MODIFIED
+
+#include <ruby.h>
+#ifdef HAVE_RUBY_ENCODING_H
+#include <ruby/encoding.h>
+#endif
+
+static VALUE charlock_new_enc_str(const char *str, size_t len, void *encoding)
+{
+#ifdef HAVE_RUBY_ENCODING_H
+	return rb_external_str_new_with_enc(str, len, (rb_encoding *)encoding);
+#else
+	return rb_str_new(str, len);
+#endif
+}
+
+static VALUE charlock_new_str(const char *str, size_t len)
+{
+#ifdef HAVE_RUBY_ENCODING_H
+	return rb_external_str_new_with_enc(str, len, rb_utf8_encoding());
+#else
+	return rb_str_new(str, len);
+#endif
+}
+
+static VALUE charlock_new_str2(const char *str)
+{
+#ifdef HAVE_RUBY_ENCODING_H
+	return rb_external_str_new_with_enc(str, strlen(str), rb_utf8_encoding());
+#else
+	return rb_str_new2(str);
+#endif
+}
+
+#endif

data/ext/charlock_holmes/converter.c

@@ -0,0 +1,54 @@
+#include "unicode/ucnv.h"
+#include "common.h"
+
+extern VALUE rb_mCharlockHolmes;
+static VALUE rb_cConverter;
+
+static VALUE rb_converter_convert(VALUE self, VALUE rb_txt, VALUE rb_src_enc, VALUE rb_dst_enc) {
+	VALUE rb_out;
+	const char *src_enc;
+	const char *dst_enc;
+	const char *src_txt;
+	char *out_buf;
+	void *rb_enc = NULL;
+	int32_t src_len;
+	int32_t out_len;
+	UErrorCode status = U_ZERO_ERROR;
+
+	src_txt = RSTRING_PTR(rb_txt);
+	src_len = RSTRING_LEN(rb_txt);
+	src_enc = RSTRING_PTR(rb_src_enc);
+	dst_enc = RSTRING_PTR(rb_dst_enc);
+
+	// first determin the size of the output buffer
+	out_len = ucnv_convert(dst_enc, src_enc, NULL, 0, src_txt, src_len, &status);
+	if (status != U_BUFFER_OVERFLOW_ERROR) {
+		rb_raise(rb_eArgError, u_errorName(status));
+	}
+	out_buf = xmalloc(out_len);
+
+	// now do the actual conversion
+	status = U_ZERO_ERROR;
+	out_len = ucnv_convert(dst_enc, src_enc, out_buf, out_len, src_txt, src_len, &status);
+	if (U_FAILURE(status)) {
+		xfree(out_buf);
+		rb_raise(rb_eArgError, u_errorName(status));
+	}
+
+	xfree(out_buf);
+
+#ifdef HAVE_RUBY_ENCODING_H
+	(rb_encoding *)rb_enc = rb_enc_find(dst_enc);
+#endif
+
+	rb_out = charlock_new_enc_str(out_buf, out_len, rb_enc);
+
+	return rb_out;
+}
+
+void _init_charlock_converter() {
+	rb_cConverter = rb_define_class_under(rb_mCharlockHolmes, "Converter", rb_cObject);
+
+	// rb_define_alloc_func(rb_cConverter, rb_converter__alloc);
+	rb_define_singleton_method(rb_cConverter, "convert", rb_converter_convert, 3);
+}

data/ext/charlock_holmes/{charlock_holmes.c → encoding_detector.c}

@@ -1,31 +1,9 @@
 #include "unicode/ucsdet.h"
+#include "common.h"
 
-#include <ruby.h>
-#ifdef HAVE_RUBY_ENCODING_H
-#include <ruby/encoding.h>
-#endif
-
-static VALUE rb_mCharlockHolmes;
+extern VALUE rb_mCharlockHolmes;
 static VALUE rb_cEncodingDetector;
 
-static VALUE charlock_new_str(const char *str, size_t len)
-{
-#ifdef HAVE_RUBY_ENCODING_H
-	return rb_external_str_new_with_enc(str, len, rb_utf8_encoding());
-#else
-	return rb_str_new(str, len);
-#endif
-}
-
-static VALUE charlock_new_str2(const char *str)
-{
-#ifdef HAVE_RUBY_ENCODING_H
-	return rb_external_str_new_with_enc(str, strlen(str), rb_utf8_encoding());
-#else
-	return rb_str_new2(str);
-#endif
-}
-
 static VALUE rb_encdec_buildmatch(const UCharsetMatch *match)
 {
 	UErrorCode status = U_ZERO_ERROR;
@@ -229,10 +207,8 @@ static VALUE rb_encdec__alloc(VALUE klass)
 	return Data_Wrap_Struct(klass, NULL, rb_encdec__free, (void *)csd);
 }
 
-void Init_charlock_holmes()
+void _init_charlock_encoding_detector()
 {
-	rb_mCharlockHolmes = rb_define_module("CharlockHolmes");
-
 	rb_cEncodingDetector = rb_define_class_under(rb_mCharlockHolmes, "EncodingDetector", rb_cObject);
 	rb_define_alloc_func(rb_cEncodingDetector, rb_encdec__alloc);
 	rb_define_method(rb_cEncodingDetector, "detect", rb_encdec_detect, -1);

data/ext/charlock_holmes/ext.c

@@ -0,0 +1,13 @@
+#include "common.h"
+
+extern void _init_charlock_encoding_detector();
+extern void _init_charlock_converter();
+
+VALUE rb_mCharlockHolmes;
+
+void Init_charlock_holmes() {
+	rb_mCharlockHolmes = rb_define_module("CharlockHolmes");
+
+	_init_charlock_encoding_detector();
+	_init_charlock_converter();
+}

data/lib/charlock_holmes/version.rb

@@ -1,3 +1,3 @@
 module CharlockHolmes
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/spec/converter_spec.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe CharlockHolmes::Converter do
+  test 'is able to convert regular ascii content from ISO-8859-1 to UTF-16, and back again' do
+    input = 'test'
+
+    output = CharlockHolmes::Converter.convert input, 'ISO-8859-1', 'UTF-16'
+    assert input.bytesize < output.bytesize
+    assert input != output
+
+    output = CharlockHolmes::Converter.convert output, 'UTF-16', 'ISO-8859-1'
+    assert input.bytesize == output.bytesize
+    assert input == output
+  end
+
+  test 'is able to convert UTF-8 content from UTF-8 to UTF-16, and back again' do
+    input = 'λ, λ, λ'
+
+    output = CharlockHolmes::Converter.convert input, 'UTF-8', 'UTF-16'
+    assert input.bytesize < output.bytesize
+    assert input != output
+
+    output = CharlockHolmes::Converter.convert output, 'UTF-16', 'UTF-8'
+    assert input.bytesize == output.bytesize
+    assert input == output
+  end
+end

metadata

@@ -1,58 +1,79 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: charlock_holmes
-version: !ruby/object:Gem::Version
-  version: 0.4.1
+version: !ruby/object:Gem::Version 
+  hash: 11
   prerelease: 
+  segments: 
+  - 0
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
-authors:
+authors: 
 - Brian Lopez
-- Vicent Martí
+- "Vicent Mart\xC3\xAD"
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-25 00:00:00.000000000 -07:00
+
+date: 2011-08-26 00:00:00 -07:00
 default_executable: 
-dependencies:
-- !ruby/object:Gem::Dependency
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: rake-compiler
-  requirement: &70218132232740 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 7
+        - 5
         version: 0.7.5
   type: :development
-  prerelease: false
-  version_requirements: *70218132232740
-- !ruby/object:Gem::Dependency
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
   name: rspec
-  requirement: &70218114472220 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 0
+        - 0
         version: 2.0.0
   type: :development
-  prerelease: false
-  version_requirements: *70218114472220
-- !ruby/object:Gem::Dependency
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
   name: chardet
-  requirement: &70218114471840 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
-  prerelease: false
-  version_requirements: *70218114471840
+  version_requirements: *id003
 description: 
 email: seniorlopez@gmail.com
 executables: []
-extensions:
+
+extensions: 
 - ext/charlock_holmes/extconf.rb
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
 - .rspec
 - Gemfile
@@ -63,12 +84,16 @@ files:
 - benchmark/detection.rb
 - benchmark/test.txt
 - charlock_holmes.gemspec
-- ext/charlock_holmes/charlock_holmes.c
+- ext/charlock_holmes/common.h
+- ext/charlock_holmes/converter.c
+- ext/charlock_holmes/encoding_detector.c
+- ext/charlock_holmes/ext.c
 - ext/charlock_holmes/extconf.rb
 - lib/charlock_holmes.rb
 - lib/charlock_holmes/encoding_detector.rb
 - lib/charlock_holmes/string.rb
 - lib/charlock_holmes/version.rb
+- spec/converter_spec.rb
 - spec/encoding_detector_spec.rb
 - spec/fixtures/AnsiGraph.psm1
 - spec/fixtures/TwigExtensionsDate.es.yml
@@ -81,31 +106,40 @@ files:
 has_rdoc: true
 homepage: http://github.com/brianmario/charlock_holmes
 licenses: []
+
 post_install_message: 
-rdoc_options:
+rdoc_options: 
 - --charset=UTF-8
-require_paths:
+require_paths: 
 - lib
 - ext
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
 rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Character encoding detection, brought to you by ICU
-test_files:
+test_files: 
+- spec/converter_spec.rb
 - spec/encoding_detector_spec.rb
 - spec/fixtures/AnsiGraph.psm1
 - spec/fixtures/TwigExtensionsDate.es.yml