charlock_holmes 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 14eb002e3883172c0808434587c7c7198e862fbe
-  data.tar.gz: ec27785739f9e4c3ca5c3c2987cf7d214415db36
+  metadata.gz: 53abf00f6c72c2ac1339b3f856011bed111b9ad4
+  data.tar.gz: c17048fa5ddf8c5c37f1378653e0bdcea6849e7a
 SHA512:
-  metadata.gz: 33d8ce4b9bbd9408459c3ab6dd4e834352b183919a45b910d74472fb2d38b898d4898cb5d2f718768192cd4b92bb102795c39784aa1238e327419051c5b47b85
-  data.tar.gz: 7f7fdcfb7c1996bf8e451cf536410d89b9c88d5bd956c292a7716f5fe119264406cacbd7872af38b2a66613233c88cc98166977da766f63398326d3176b8f945
+  metadata.gz: 4f97cf5d2bca0e320e1eb54efd77bd4efed5add1621a4c9dcd62e6ec0c1b0b0834a30a5247341308974683e93a064044c2fb524b562d88dd26ead19694bf2121
+  data.tar.gz: 57ac5e9d12ae54f65387ef81f45afde88a8d4988f38f2647e2fbc3c88fe00fe094361f8eb26120f7bacea21024f73a74118bfc06a762d7fcc47caecfd5edc372

data/ext/charlock_holmes/encoding_detector.c

@@ -15,6 +15,9 @@ static VALUE rb_encdec_buildmatch(const UCharsetMatch *match)
 	const char *mlang;
 	int mconfidence;
 	VALUE rb_match;
+	VALUE enc_tbl;
+	VALUE enc_name;
+	VALUE compat_enc;
 
 	if (!match)
 		return Qnil;
@@ -26,7 +29,16 @@ static VALUE rb_encdec_buildmatch(const UCharsetMatch *match)
 	rb_match = rb_hash_new();
 
 	rb_hash_aset(rb_match, ID2SYM(rb_intern("type")), ID2SYM(rb_intern("text")));
-	rb_hash_aset(rb_match, ID2SYM(rb_intern("encoding")), charlock_new_str2(mname));
+
+	enc_name = charlock_new_str2(mname);
+	rb_hash_aset(rb_match, ID2SYM(rb_intern("encoding")), enc_name);
+
+	enc_tbl = rb_iv_get(rb_cEncodingDetector, "@encoding_table");
+	compat_enc = rb_hash_aref(enc_tbl, enc_name);
+	if (!NIL_P(compat_enc)) {
+		rb_hash_aset(rb_match, ID2SYM(rb_intern("ruby_encoding")), compat_enc);
+	}
+
 	rb_hash_aset(rb_match, ID2SYM(rb_intern("confidence")), INT2NUM(mconfidence));
 
 	if (mlang && mlang[0])

data/lib/charlock_holmes/encoding_detector.rb

@@ -41,5 +41,36 @@ module CharlockHolmes
     def self.detect_all(str, hint_enc=nil)
       new.detect_all(str, hint_enc)
     end
+
+    # A mapping table of supported encoding names from EncodingDetector
+    # which point to the corresponding supported encoding name in Ruby.
+    # Like: {"UTF-8" => "UTF-8", "IBM420_rtl" => "ASCII-8BIT"}
+    #
+    # Note that encodings that can't be mapped between Charlock and Ruby will resolve
+    # to "ASCII-8BIT".
+    @encoding_table = {}
+
+    def self.encoding_table
+      @encoding_table
+    end
+
+    BINARY = 'binary'
+
+    # Builds the ENCODING_TABLE hash by running through the list of supported encodings
+    # in the ICU detection API and trying to map them to supported encodings in Ruby.
+    # This is built dynamically so as to take advantage of ICU upgrades which may have
+    # support for more encodings in the future.
+    #
+    # Returns nothing.
+    def self.build_encoding_table
+      supported_encodings.each do |name|
+        @encoding_table[name] = begin
+          ::Encoding.find(name).name
+        rescue ArgumentError
+          BINARY
+        end
+      end
+    end
+    build_encoding_table
   end
 end

data/lib/charlock_holmes/string.rb

@@ -26,7 +26,7 @@ class String
     # Returns: self
     def detect_encoding!(hint_enc=nil)
       if detected = self.detect_encoding(hint_enc)
-        self.force_encoding(detected[:encoding]) if detected[:encoding]
+        self.force_encoding(detected[:ruby_encoding]) if detected[:ruby_encoding]
       end
       self
     end

data/lib/charlock_holmes/version.rb

@@ -1,3 +1,3 @@
 module CharlockHolmes
-  VERSION = "0.7.1"
+  VERSION = "0.7.2"
 end

data/test/encoding_detector_test.rb

@@ -89,6 +89,17 @@ class EncodingDetectorTest < MiniTest::Test
     assert supported_encodings.include? 'UTF-8'
   end
 
+  def test_returns_a_ruby_compatible_encoding_name
+    detected = @detector.detect 'test'
+    assert_equal 'ISO-8859-1', detected[:encoding]
+    assert_equal 'ISO-8859-1', detected[:ruby_encoding]
+
+    not_compat_txt = fixture("ISO-2022-KR.txt").read
+    detected = @detector.detect not_compat_txt
+    assert_equal 'ISO-2022-KR', detected[:encoding]
+    assert_equal 'binary', detected[:ruby_encoding]
+  end
+
   MAPPING = [
     ['repl2.cljs',                'ISO-8859-1', :text],
     ['cl-messagepack.lisp',       'ISO-8859-1', :text],
@@ -114,8 +125,7 @@ class EncodingDetectorTest < MiniTest::Test
     MAPPING.each do |mapping|
       file, encoding, type = mapping
 
-      path = File.expand_path "../fixtures/#{file}", __FILE__
-      content = File.read path
+      content = fixture(file).read
       guessed = @detector.detect content
 
       assert_equal encoding, guessed[:encoding]

data/test/fixtures/ISO-2022-KR.txt

@@ -0,0 +1,43 @@
+$)C#
+# Out-AnsiGraph.psm1
+# Author:       xcud
+# History:
+#       v0.1 September 21, 2009 initial version
+#
+# PS Example> ps | select -first 5 | sort -property VM | 
+#             Out-AnsiGraph ProcessName, VM
+#                 AEADISRV  14508032
+#                  audiodg  50757632
+#                  conhost  73740288
+# AppleMobileDeviceService  92061696
+#                    btdna  126443520
+#
+function Out-AnsiGraph($Parameter1=$null) {
+	BEGIN {
+		$q = new-object Collections.queue
+		$max = 0; $namewidth = 0;
+	}
+
+	PROCESS {
+		if($_) {
+			$name = $_.($Parameter1[0]);
+			$val = $_.($Parameter1[1])
+			if($max -lt $val) { $max = $val}		 
+			if($namewidth -lt $name.length) { 
+				$namewidth = $name.length }
+			$q.enqueue(@($name, $val))			
+		}
+	}
+
+	END {
+		$q | %{
+			$graph = ""; 0..($_[1]/$max*20) | 
+				%{ $graph += "" }
+			$name = "{0,$namewidth}" -f $_[0]
+			"$name $graph " + $_[1]
+		}
+
+	}
+}
+
+Export-ModuleMember Out-AnsiGraph

data/test/helper.rb

@@ -16,6 +16,11 @@ else
   Minitest::Test = MiniTest::Unit::TestCase
 end
 
+def fixture(name)
+  path = File.expand_path "../fixtures/#{name}", __FILE__
+  File.new path
+end
+
 # put lib and test dirs directly on load path
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
-$LOAD_PATH.unshift File.expand_path('..', __FILE__)
+$LOAD_PATH.unshift File.expand_path('..', __FILE__)

data/test/string_methods_test.rb

@@ -40,7 +40,18 @@ class StringMethodsTest < MiniTest::Test
     assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
   end
 
-  if RUBY_VERSION =~ /1.9/
+  def test_returns_a_ruby_compatible_encoding_name
+    detected = 'test'.detect_encoding
+    assert_equal 'ISO-8859-1', detected[:encoding]
+    assert_equal 'ISO-8859-1', detected[:ruby_encoding]
+
+    not_compat_txt = fixture("ISO-2022-KR.txt").read
+    detected = not_compat_txt.detect_encoding
+    assert_equal 'ISO-2022-KR', detected[:encoding]
+    assert_equal 'binary', detected[:ruby_encoding]
+  end
+
+  if "".respond_to? :force_encoding
     def test_adds_detect_encoding_bang_method
       str = 'test'
       str.respond_to? :detect_encoding!
@@ -48,5 +59,15 @@ class StringMethodsTest < MiniTest::Test
       str.detect_encoding!
       assert_equal Encoding.find('ISO-8859-1'), str.encoding
     end
+
+    def test_sets_a_ruby_compatible_encoding_name
+      str1 = 'test'
+      str1.detect_encoding!
+      assert_equal 'ISO-8859-1', str1.encoding.name
+
+      not_compat_txt = fixture("ISO-2022-KR.txt").read
+      not_compat_txt.detect_encoding!
+      assert_equal 'ASCII-8BIT', not_compat_txt.encoding.name
+    end
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: charlock_holmes
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Brian Lopez
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-12 00:00:00.000000000 Z
+date: 2014-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -82,6 +82,7 @@ files:
 - test/converter_test.rb
 - test/encoding_detector_test.rb
 - test/fixtures/AnsiGraph.psm1
+- test/fixtures/ISO-2022-KR.txt
 - test/fixtures/TwigExtensionsDate.es.yml
 - test/fixtures/cl-messagepack.lisp
 - test/fixtures/core.rkt