charlock_holmes 0.5.0 → 0.6.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    charlock_holmes (0.5.0)
+    charlock_holmes (0.6.0)
 
 GEM
   remote: http://rubygems.org/

data/README.md

@@ -10,12 +10,12 @@ First you'll need to require it
 require 'charlock_holmes'
 ```
 
-Now if you want to detect the encoding of a set of bytes:
+## Encoding detection
 
 ``` ruby
 contents = File.read('test.xml')
 detection = CharlockHolmes::EncodingDetector.detect(contents)
-# => {:encoding => 'UTF-8', :confidence => 100}
+# => {:encoding => 'UTF-8', :confidence => 100, :type => :text}
 
 # optionally there will be a :language key as well, but
 # that's mostly only returned for legacy encodings like ISO-8859-1
@@ -23,6 +23,8 @@ detection = CharlockHolmes::EncodingDetector.detect(contents)
 
 NOTE: `CharlockHolmes::EncodingDetector.detect` will return `nil` if it was unable to find an encoding.
 
+For binary content, `:type` will be set to `:binary`
+
 Though it's more efficient to reuse once detector instance:
 
 ``` ruby
@@ -34,7 +36,7 @@ detection2 = detector.detect(File.read('test2.json'))
 # and so on...
 ```
 
-## String monkey patch
+### String monkey patch
 
 Alternatively, you can just use the `detect_encoding` method on the `String` class
 
@@ -46,7 +48,7 @@ contents = File.read('test.xml')
 detection = contents.detect_encoding
 ```
 
-## Ruby 1.9 specific
+### Ruby 1.9 specific
 
 NOTE: This method only exists on Ruby 1.9+
 
@@ -57,18 +59,30 @@ require 'charlock_holmes/string'
 
 contents = File.read('test.xml')
 
-# this will detect and set the encoding of `contents`
+# this will detect and set the encoding of `contents`, then return self
 contents.detect_encoding!
 ```
 
+## Transcoding
+
+Being able to detect the encoding of some arbitrary content is nice, but what you probably want is to be able to transcode that content into an encoding your application is using.
+
+``` ruby
+content = File.read('test2.txt')
+detection = CharlockHolmes::EncodingDetector.detect(content)
+utf8_encoded_content CharlockHolmes::Converter.convert content, detection[:encoding], 'UTF-8'
+```
+
+The first parameter is the content to transcode, the second is the source encoding (the encoding the content is assumed to be in), and the third parameter is the destination encoding.
+
 ## Installing
 
-If the traditional `gem install charlock_holmes` doesn't work, you may need to specify the path to your installation of ICU using the `--with-icu-dir` option during the gem install.
+If the traditional `gem install charlock_holmes` doesn't work, you may need to specify the path to your installation of ICU and libmagic using the `--with-icu-dir` and/or `--with-magic-dir` option during the gem install.
 
-At the time of writing, if you installed ICU via homebrew on OSX your gem install may look something like this:
+At the time of writing, Homebrew for OSX installs ICU (icu4c is the package name) and libmagic as keg-only installs so you'll have to specify the location during the gem install:
 
-`gem install charlock_holmes --with-icu-dir=/usr/local/Cellar/icu4c/4.4.1`
+`gem install charlock_holmes --with-icu-dir=/usr/local/Cellar/icu4c/4.4.1 --with-magic-dir=/usr/local/Cellar/libmagic/5.04`
 
-If you're using Bundler and need to specify a custom path, you can do so with the `bundle config` command:
+If you're using Bundler and need to specify a custom path(s), you can do so with the `bundle config` command:
 
-`bundle config build.charlock_holmes --with-icu-dir=/usr/local/Cellar/icu4c/4.4.1`
+`bundle config build.charlock_holmes --with-icu-dir=/usr/local/Cellar/icu4c/4.4.1 --with-magic-dir=/usr/local/Cellar/libmagic/5.04`

data/benchmark/detection.rb

@@ -16,14 +16,14 @@ DETECTOR = CharlockHolmes::EncodingDetector.new
 
 Benchmark.bmbm do |x|
   # new detector every iteration
-  x.report 'class-level detect' do
+  x.report 'singleton call' do
     TIMES.times do
       CharlockHolmes::EncodingDetector.detect CONTENT
     end
   end
 
   # shared detector for all iterations
-  x.report 'instance-level detect' do
+  x.report 'reusing a single detector' do
     TIMES.times do
       DETECTOR.detect CONTENT
     end

data/ext/charlock_holmes/converter.c

@@ -49,6 +49,5 @@ static VALUE rb_converter_convert(VALUE self, VALUE rb_txt, VALUE rb_src_enc, VA
 void _init_charlock_converter() {
 	rb_cConverter = rb_define_class_under(rb_mCharlockHolmes, "Converter", rb_cObject);
 
-	// rb_define_alloc_func(rb_cConverter, rb_converter__alloc);
 	rb_define_singleton_method(rb_cConverter, "convert", rb_converter_convert, 3);
 }

data/ext/charlock_holmes/encoding_detector.c

@@ -1,9 +1,15 @@
 #include "unicode/ucsdet.h"
+#include "magic.h"
 #include "common.h"
 
 extern VALUE rb_mCharlockHolmes;
 static VALUE rb_cEncodingDetector;
 
+typedef struct {
+	UCharsetDetector *csd;
+	magic_t magic;
+} charlock_detector_t;
+
 static VALUE rb_encdec_buildmatch(const UCharsetMatch *match)
 {
 	UErrorCode status = U_ZERO_ERROR;
@@ -21,6 +27,7 @@ static VALUE rb_encdec_buildmatch(const UCharsetMatch *match)
 
 	rb_match = rb_hash_new();
 
+	rb_hash_aset(rb_match, ID2SYM(rb_intern("type")), ID2SYM(rb_intern("text")));
 	rb_hash_aset(rb_match, ID2SYM(rb_intern("encoding")), charlock_new_str2(mname));
 	rb_hash_aset(rb_match, ID2SYM(rb_intern("confidence")), INT2NUM(mconfidence));
 
@@ -30,6 +37,36 @@ static VALUE rb_encdec_buildmatch(const UCharsetMatch *match)
 	return rb_match;
 }
 
+static VALUE rb_encdec_binarymatch() {
+	VALUE rb_match;
+
+	rb_match = rb_hash_new();
+
+	rb_hash_aset(rb_match, ID2SYM(rb_intern("type")), ID2SYM(rb_intern("binary")));
+	rb_hash_aset(rb_match, ID2SYM(rb_intern("confidence")), INT2NUM(100));
+
+	return rb_match;
+}
+
+static int detect_binary_content(charlock_detector_t *detector, VALUE rb_str) {
+	const char *binary_result;
+
+	binary_result = magic_buffer(detector->magic, RSTRING_PTR(rb_str), RSTRING_LEN(rb_str));
+
+	if (binary_result) {
+		if (strstr(binary_result, "library") ||
+				strstr(binary_result, "bundle") ||
+				strstr(binary_result, "archive") ||
+				(!strstr(binary_result, "text") && strstr(binary_result, "executable")) ||
+				strstr(binary_result, "data"))
+			return 1;
+	} else {
+		rb_raise(rb_eStandardError, magic_error(detector->magic));
+	}
+
+	return 0;
+}
+
 /*
  * call-seq: detection_hash = EncodingDetector.detect str[, hint_enc]
  *
@@ -39,28 +76,35 @@ static VALUE rb_encdec_buildmatch(const UCharsetMatch *match)
  * hint_enc - an optional String (like "UTF-8"), the encoding name which will
  *            be used as an additional hint to the charset detector
  *
- * Returns: a Hash with :encoding, :language and :confidence
+ * Returns: a Hash with :encoding, :language, :type and :confidence
  */
 static VALUE rb_encdec_detect(int argc, VALUE *argv, VALUE self)
 {
 	UErrorCode status = U_ZERO_ERROR;
-	UCharsetDetector *csd;
+	charlock_detector_t *detector;
 	VALUE rb_str;
 	VALUE rb_enc_hint;
 
 	rb_scan_args(argc, argv, "11", &rb_str, &rb_enc_hint);
 
 	Check_Type(rb_str, T_STRING);
-	Data_Get_Struct(self, UCharsetDetector, csd);
+	Data_Get_Struct(self, charlock_detector_t, detector);
 
-	ucsdet_setText(csd, RSTRING_PTR(rb_str), (int32_t)RSTRING_LEN(rb_str), &status);
+	// first lets see if this is binary content
+	if (detect_binary_content(detector, rb_str)) {
+		return rb_encdec_binarymatch();
+	}
+
+	// if we got here - the data doesn't look like binary
+	// lets try to figure out what encoding the text is in
+	ucsdet_setText(detector->csd, RSTRING_PTR(rb_str), (int32_t)RSTRING_LEN(rb_str), &status);
 
 	if (!NIL_P(rb_enc_hint)) {
 		Check_Type(rb_enc_hint, T_STRING);
-		ucsdet_setDeclaredEncoding(csd, RSTRING_PTR(rb_enc_hint), RSTRING_LEN(rb_enc_hint), &status);
+		ucsdet_setDeclaredEncoding(detector->csd, RSTRING_PTR(rb_enc_hint), RSTRING_LEN(rb_enc_hint), &status);
 	}
 
-	return rb_encdec_buildmatch(ucsdet_detect(csd, &status));
+	return rb_encdec_buildmatch(ucsdet_detect(detector->csd, &status));
 }
 
 
@@ -76,38 +120,48 @@ static VALUE rb_encdec_detect(int argc, VALUE *argv, VALUE self)
  *            be used as an additional hint to the charset detector
  *
  * Returns: an Array with zero or more Hashes,
- *          each one of them with with :encoding, :language and :confidence
+ *          each one of them with with :encoding, :language, :type and :confidence
  */
 static VALUE rb_encdec_detect_all(int argc, VALUE *argv, VALUE self)
 {
 	UErrorCode status = U_ZERO_ERROR;
-	UCharsetDetector *csd;
+	charlock_detector_t *detector;
 	const UCharsetMatch **csm;
 	VALUE rb_ret;
 	int i, match_count;
 	VALUE rb_str;
 	VALUE rb_enc_hint;
+	VALUE binary_match;
 
 	rb_scan_args(argc, argv, "11", &rb_str, &rb_enc_hint);
 
 	Check_Type(rb_str, T_STRING);
-	Data_Get_Struct(self, UCharsetDetector, csd);
+	Data_Get_Struct(self, charlock_detector_t, detector);
 
 	rb_ret = rb_ary_new();
 
-	ucsdet_setText(csd, RSTRING_PTR(rb_str), (int32_t)RSTRING_LEN(rb_str), &status);
+	// first lets see if this is binary content
+	binary_match = Qnil;
+	if (detect_binary_content(detector, rb_str)) {
+		binary_match = rb_encdec_binarymatch();
+	}
+
+	ucsdet_setText(detector->csd, RSTRING_PTR(rb_str), (int32_t)RSTRING_LEN(rb_str), &status);
 
 	if (!NIL_P(rb_enc_hint)) {
 		Check_Type(rb_enc_hint, T_STRING);
-		ucsdet_setDeclaredEncoding(csd, RSTRING_PTR(rb_enc_hint), RSTRING_LEN(rb_enc_hint), &status);
+		ucsdet_setDeclaredEncoding(detector->csd, RSTRING_PTR(rb_enc_hint), RSTRING_LEN(rb_enc_hint), &status);
 	}
 
-	csm = ucsdet_detectAll(csd, &match_count, &status);
+	csm = ucsdet_detectAll(detector->csd, &match_count, &status);
 
 	for (i = 0; i < match_count; ++i) {
 		rb_ary_push(rb_ret, rb_encdec_buildmatch(csm[i]));
 	}
 
+	if (!NIL_P(binary_match))
+		rb_ary_unshift(rb_ret, binary_match);
+
 	return rb_ret;
 }
 
@@ -120,13 +174,13 @@ static VALUE rb_encdec_detect_all(int argc, VALUE *argv, VALUE self)
  */
 static VALUE rb_get_strip_tags(VALUE self)
 {
-	UCharsetDetector *csd;
+	charlock_detector_t *detector;
 	UBool val;
 	VALUE rb_val;
 
-	Data_Get_Struct(self, UCharsetDetector, csd);
+	Data_Get_Struct(self, charlock_detector_t, detector);
 
-	val = ucsdet_isInputFilterEnabled(csd);
+	val = ucsdet_isInputFilterEnabled(detector->csd);
 
 	rb_val = val == 1 ? Qtrue : Qfalse;
 
@@ -143,14 +197,14 @@ static VALUE rb_get_strip_tags(VALUE self)
  */
 static VALUE rb_set_strip_tags(VALUE self, VALUE rb_val)
 {
-	UCharsetDetector *csd;
+	charlock_detector_t *detector;
 	UBool val;
 
-	Data_Get_Struct(self, UCharsetDetector, csd);
+	Data_Get_Struct(self, charlock_detector_t, detector);
 
 	val = rb_val == Qtrue ? 1 : 0;
 
-	ucsdet_enableInputFilter(csd, val);
+	ucsdet_enableInputFilter(detector->csd, val);
 
 	return rb_val;
 }
@@ -195,16 +249,42 @@ static VALUE rb_get_supported_encodings(VALUE klass)
 	return rb_encoding_list;
 }
 
-static void rb_encdec__free(void *csd)
+static void rb_encdec__free(void *obj)
 {
-	ucsdet_close((UCharsetDetector *)csd);
+	charlock_detector_t *detector;
+
+	detector = (charlock_detector_t *)obj;
+
+	if (detector->csd)
+		ucsdet_close(detector->csd);
+
+	if (detector->magic)
+		magic_close(detector->magic);
 }
 
 static VALUE rb_encdec__alloc(VALUE klass)
 {
+	charlock_detector_t *detector;
 	UErrorCode status = U_ZERO_ERROR;
-	UCharsetDetector *csd = ucsdet_open(&status);
-	return Data_Wrap_Struct(klass, NULL, rb_encdec__free, (void *)csd);
+	VALUE obj;
+
+	obj = Data_Make_Struct(klass, charlock_detector_t, NULL, rb_encdec__free, (void *)detector);
+
+	detector->csd = ucsdet_open(&status);
+	if (U_FAILURE(status)) {
+		rb_raise(rb_eStandardError, u_errorName(status));
+	}
+
+	detector->magic = magic_open(0);
+	if (detector->magic == NULL) {
+		rb_raise(rb_eStandardError, magic_error(detector->magic));
+	}
+
+	// load the libmagic database
+	// NULL means use the default or whatever is specified by the MAGIC env var
+	magic_load(detector->magic, NULL);
+
+	return obj;
 }
 
 void _init_charlock_encoding_detector()

data/ext/charlock_holmes/extconf.rb

@@ -3,11 +3,14 @@ require 'mkmf'
 $CFLAGS << ' -Wall -funroll-loops'
 $CFLAGS << ' -Wextra -O0 -ggdb3' if ENV['DEBUG']
 
+dir_config 'icu'
 $CFLAGS  << ' -I/usr/local/Cellar/icu4c/4.4.1/include'
 $LDFLAGS << ' -L/usr/local/Cellar/icu4c/4.4.1/lib'
-
-dir_config 'icu'
-
 have_library 'icui18n'
 
+dir_config 'magic'
+$LDFLAGS << ' -L/usr/local/Cellar/libmagic/5.04/lib'
+$CFLAGS  << ' -I/usr/local/Cellar/libmagic/5.04/include'
+have_library 'magic'
+
 create_makefile 'charlock_holmes'

data/lib/charlock_holmes/encoding_detector.rb

@@ -10,7 +10,7 @@ module CharlockHolmes
     # hint_enc - an optional String (like "UTF-8"), the encoding name which will
     #            be used as an additional hint to the charset detector
     #
-    # Returns: a Hash with :encoding, :language and :confidence
+    # Returns: a Hash with :encoding, :language, :type and :confidence
     def self.detect(str, hint_enc=nil)
       new.detect(str, hint_enc)
     end
@@ -25,7 +25,7 @@ module CharlockHolmes
     #            be used as an additional hint to the charset detector
     #
     # Returns: an Array with zero or more Hashes,
-    # each one of them with with :encoding, :language and :confidence
+    # each one of them with with :encoding, :language, :type and :confidence
     def self.detect_all(str, hint_enc=nil)
       new.detect_all(str, hint_enc)
     end

data/lib/charlock_holmes/string.rb

@@ -3,7 +3,7 @@ require 'charlock_holmes' unless defined? CharlockHolmes
 class String
   # Attempt to detect the encoding of this string
   #
-  # Returns: a Hash with :encoding, :language and :confidence
+  # Returns: a Hash with :encoding, :language, :type and :confidence
   def detect_encoding(hint_enc=nil)
     encoding_detector.detect(self, hint_enc)
   end
@@ -12,7 +12,7 @@ class String
   # a list with all the possible encodings that match it.
   #
   # Returns: an Array with zero or more Hashes,
-  #          each one of them with with :encoding, :language and :confidence
+  #          each one of them with with :encoding, :language, :type and :confidence
   def detect_encodings(hint_enc=nil)
     encoding_detector.detect_all(self, hint_enc)
   end
@@ -21,12 +21,12 @@ class String
     # Attempt to detect the encoding of this string
     # then set the encoding to what was detected ala `force_encoding`
     #
-    # Returns: a Hash with :encoding, :language and :confidence
+    # Returns: self
     def detect_encoding!(hint_enc=nil)
       if detected = self.detect_encoding(hint_enc)
         self.force_encoding detected[:encoding]
-        detected
       end
+      self
     end
   end
 

data/lib/charlock_holmes/version.rb

@@ -1,3 +1,3 @@
 module CharlockHolmes
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/spec/encoding_detector_spec.rb

@@ -92,16 +92,17 @@ describe CharlockHolmes::EncodingDetector do
 
   context 'encoding detection' do
     MAPPING = [
-      ['repl2.cljs',                'ISO-8859-1'],
-      ['core.rkt',                  'UTF-8'],
-      ['cl-messagepack.lisp',       'ISO-8859-1'],
-      ['TwigExtensionsDate.es.yml', 'UTF-8'],
-      ['AnsiGraph.psm1',            'UTF-16LE'],
-      ['laholator.py',              'UTF-8']
+      ['repl2.cljs',                'ISO-8859-1', :text],
+      ['core.rkt',                  'UTF-8',      :text],
+      ['cl-messagepack.lisp',       'ISO-8859-1', :text],
+      ['TwigExtensionsDate.es.yml', 'UTF-8',      :text],
+      ['AnsiGraph.psm1',            'UTF-16LE',   :text],
+      ['laholator.py',              'UTF-8',      :text],
+      ['hello_world',               nil,          :binary]
     ]
 
     MAPPING.each do |mapping|
-      file, encoding = mapping
+      file, encoding, type = mapping
 
       test "#{file} should be detected as #{encoding}" do
         path = File.expand_path "../fixtures/#{file}", __FILE__
@@ -109,6 +110,7 @@ describe CharlockHolmes::EncodingDetector do
         guessed = @detector.detect content
 
         assert_equal encoding, guessed[:encoding]
+        assert_equal type, guessed[:type]
 
         if content.respond_to? :force_encoding
           content.force_encoding guessed[:encoding]

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: charlock_holmes
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 7
   prerelease: 
   segments: 
   - 0
-  - 5
+  - 6
   - 0
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Brian Lopez
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-26 00:00:00 -07:00
+date: 2011-08-27 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -99,6 +99,7 @@ files:
 - spec/fixtures/TwigExtensionsDate.es.yml
 - spec/fixtures/cl-messagepack.lisp
 - spec/fixtures/core.rkt
+- spec/fixtures/hello_world
 - spec/fixtures/laholator.py
 - spec/fixtures/repl2.cljs
 - spec/spec_helper.rb
@@ -145,6 +146,7 @@ test_files:
 - spec/fixtures/TwigExtensionsDate.es.yml
 - spec/fixtures/cl-messagepack.lisp
 - spec/fixtures/core.rkt
+- spec/fixtures/hello_world
 - spec/fixtures/laholator.py
 - spec/fixtures/repl2.cljs
 - spec/spec_helper.rb