charlock_holmes 0.2.0 → 0.3.0

data/README.md

@@ -21,6 +21,8 @@ detection = CharlockHolmes::EncodingDetector.detect(contents)
 # that's mostly only returned for legacy encodings like ISO-8859-1
 ```
 
+NOTE: `CharlockHolmes::EncodingDetector.detect` will return `nil` if it was unable to find an encoding.
+
 Though it's more efficient to reuse once detector instance:
 
 ``` ruby
@@ -66,3 +68,7 @@ If the traditional `gem install charlock_holmes` doesn't work, you may need to s
 At the time of writing, if you installed ICU via homebrew on OSX your gem install may look something like this:
 
 `gem install charlock_holmes --with-icu-dir=/usr/local/Cellar/icu4c/4.4.1`
+
+If you're using Bundler and need to specify a custom path, you can do so with the `bundle config` command:
+
+`bundle config build.charlock_holmes --with-icu-dir=/usr/local/Cellar/icu4c/4.4.1`

data/ext/charlock_holmes/charlock_holmes.c

@@ -44,41 +44,64 @@ static VALUE rb_encdec_buildmatch(const UCharsetMatch *match)
 }
 
 /*
- * call-seq: detection_hash = EncodingDetector.detect "some string"
+ * call-seq: detection_hash = EncodingDetector.detect str[, hint_enc]
  *
  * Attempt to detect the encoding of this string
  *
+ * str      - a String, what you want to detect the encoding of
+ * hint_enc - an optional String (like "UTF-8"), the encoding name which will
+ *            be used as an additional hint to the charset detector
+ *
  * Returns: a Hash with :encoding, :language and :confidence
  */
-static VALUE rb_encdec_detect(VALUE self, VALUE rb_str)
+static VALUE rb_encdec_detect(int argc, VALUE *argv, VALUE self)
 {
 	UErrorCode status = U_ZERO_ERROR;
 	UCharsetDetector *csd;
+	VALUE rb_str;
+	VALUE rb_enc_hint;
+
+	rb_scan_args(argc, argv, "11", &rb_str, &rb_enc_hint);
 
 	Check_Type(rb_str, T_STRING);
 	Data_Get_Struct(self, UCharsetDetector, csd);
 
 	ucsdet_setText(csd, RSTRING_PTR(rb_str), (int32_t)RSTRING_LEN(rb_str), &status);
+
+	if (!NIL_P(rb_enc_hint)) {
+		Check_Type(rb_enc_hint, T_STRING);
+		ucsdet_setDeclaredEncoding(csd, RSTRING_PTR(rb_enc_hint), RSTRING_LEN(rb_enc_hint), &status);
+	}
+
 	return rb_encdec_buildmatch(ucsdet_detect(csd, &status));
 }
 
 
 /*
- * call-seq: detection_hash_array = EncodingDetector.detect_all "some string"
+ * call-seq: detection_hash_array = EncodingDetector.detect_all str[, hint_enc]
  *
  * Attempt to detect the encoding of this string, and return
  * a list with all the possible encodings that match it.
  *
- * Returns: a List with zero or more Hashes,
+ *
+ * str      - a String, what you want to detect the encoding of
+ * hint_enc - an optional String (like "UTF-8"), the encoding name which will
+ *            be used as an additional hint to the charset detector
+ *
+ * Returns: an Array with zero or more Hashes,
  *	each one of them with with :encoding, :language and :confidence
  */
-static VALUE rb_encdec_detect_all(VALUE self, VALUE rb_str)
+static VALUE rb_encdec_detect_all(int argc, VALUE *argv, VALUE self)
 {
 	UErrorCode status = U_ZERO_ERROR;
 	UCharsetDetector *csd;
-    const UCharsetMatch **csm;
+	const UCharsetMatch **csm;
 	VALUE rb_ret;
 	int i, match_count;
+	VALUE rb_str;
+	VALUE rb_enc_hint;
+
+	rb_scan_args(argc, argv, "11", &rb_str, &rb_enc_hint);
 
 	Check_Type(rb_str, T_STRING);
 	Data_Get_Struct(self, UCharsetDetector, csd);
@@ -86,7 +109,13 @@ static VALUE rb_encdec_detect_all(VALUE self, VALUE rb_str)
 	rb_ret = rb_ary_new();
 
 	ucsdet_setText(csd, RSTRING_PTR(rb_str), (int32_t)RSTRING_LEN(rb_str), &status);
-    csm = ucsdet_detectAll(csd, &match_count, &status);
+
+	if (!NIL_P(rb_enc_hint)) {
+		Check_Type(rb_enc_hint, T_STRING);
+		ucsdet_setDeclaredEncoding(csd, RSTRING_PTR(rb_enc_hint), RSTRING_LEN(rb_enc_hint), &status);
+	}
+
+	csm = ucsdet_detectAll(csd, &match_count, &status);
 
 	for (i = 0; i < match_count; ++i) {
 		rb_ary_push(rb_ret, rb_encdec_buildmatch(csm[i]));
@@ -114,6 +143,6 @@ void Init_charlock_holmes()
 
 	rb_cEncodingDetector = rb_define_class_under(rb_mCharlockHolmes, "EncodingDetector", rb_cObject);
 	rb_define_alloc_func(rb_cEncodingDetector, rb_encdec__alloc);
-	rb_define_method(rb_cEncodingDetector, "detect", rb_encdec_detect, 1);
-	rb_define_method(rb_cEncodingDetector, "detect_all", rb_encdec_detect_all, 1);
+	rb_define_method(rb_cEncodingDetector, "detect", rb_encdec_detect, -1);
+	rb_define_method(rb_cEncodingDetector, "detect_all", rb_encdec_detect_all, -1);
 }

data/ext/charlock_holmes/extconf.rb

@@ -3,6 +3,9 @@ require 'mkmf'
 $CFLAGS << ' -Wall -funroll-loops'
 $CFLAGS << ' -Wextra -O0 -ggdb3' if ENV['DEBUG']
 
+$CFLAGS  << ' -I/usr/local/Cellar/icu4c/4.4.1/include'
+$LDFLAGS << ' -L/usr/local/Cellar/icu4c/4.4.1/lib'
+
 dir_config 'icu'
 
 have_library 'icui18n'

data/lib/charlock_holmes/encoding_detector.rb

@@ -4,9 +4,28 @@ module CharlockHolmes
     #
     # NOTE: This will create a new CharlockHolmes::EncodingDetector instance on every call
     #
+    # str      - a String, what you want to detect the encoding of
+    # hint_enc - an optional String (like "UTF-8"), the encoding name which will
+    #            be used as an additional hint to the charset detector
+    #
     # Returns: a Hash with :encoding, :language and :confidence
-    def self.detect(str)
-      new.detect(str)
+    def self.detect(str, hint_enc=nil)
+      new.detect(str, hint_enc)
+    end
+
+    # Attempt to detect the encoding of this string, and return
+    # a list with all the possible encodings that match it.
+    #
+    # NOTE: This will create a new CharlockHolmes::EncodingDetector instance on every call
+    #
+    # str      - a String, what you want to detect the encoding of
+    # hint_enc - an optional String (like "UTF-8"), the encoding name which will
+    #            be used as an additional hint to the charset detector
+    #
+    # Returns: an Array with zero or more Hashes,
+    # each one of them with with :encoding, :language and :confidence
+    def self.detect_all(str, hint_enc=nil)
+      new.detect_all(str, hint_enc)
     end
   end
 end

data/lib/charlock_holmes/string.rb

@@ -4,8 +4,17 @@ class String
   # Attempt to detect the encoding of this string
   #
   # Returns: a Hash with :encoding, :language and :confidence
-  def detect_encoding
-    encoding_detector.detect(self)
+  def detect_encoding(hint_enc=nil)
+    encoding_detector.detect(self, hint_enc)
+  end
+
+  # Attempt to detect the encoding of this string, and return
+  # a list with all the possible encodings that match it.
+  #
+  # Returns: an Array with zero or more Hashes,
+  #          each one of them with with :encoding, :language and :confidence
+  def detect_encodings(hint_enc=nil)
+    encoding_detector.detect_all(self, hint_enc)
   end
 
   if RUBY_VERSION =~ /1.9/
@@ -13,8 +22,8 @@ class String
     # then set the encoding to what was detected ala `force_encoding`
     #
     # Returns: a Hash with :encoding, :language and :confidence
-    def detect_encoding!
-      if detected = self.detect_encoding
+    def detect_encoding!(hint_enc=nil)
+      if detected = self.detect_encoding(hint_enc)
         self.force_encoding detected[:encoding]
         detected
       end

data/lib/charlock_holmes/version.rb

@@ -1,3 +1,3 @@
 module CharlockHolmes
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/spec/encoding_detector_spec.rb

@@ -5,21 +5,62 @@ describe CharlockHolmes::EncodingDetector do
     @detector = CharlockHolmes::EncodingDetector.new
   end
 
-  test 'has a detect class-level method' do
+  test 'has a class-level detect method' do
     CharlockHolmes::EncodingDetector.respond_to? :detect
     detected = CharlockHolmes::EncodingDetector.detect 'test'
     assert_equal 'ISO-8859-1', detected[:encoding]
   end
 
+  test 'has a class-level detect method that accepts an encoding hint' do
+    CharlockHolmes::EncodingDetector.respond_to? :detect
+    detected = CharlockHolmes::EncodingDetector.detect 'test', 'UTF-8'
+    assert_equal 'ISO-8859-1', detected[:encoding]
+  end
+
+  test 'has a class-level detect_all method' do
+    CharlockHolmes::EncodingDetector.respond_to? :detect_all
+    detected_list = CharlockHolmes::EncodingDetector.detect_all 'test'
+    assert detected_list.is_a? Array
+
+    encoding_list = detected_list.map {|d| d[:encoding]}.sort
+    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
+  end
+
+  test 'has a class-level detect_all method that accepts an encoding hint' do
+    CharlockHolmes::EncodingDetector.respond_to? :detect_all
+    detected_list = CharlockHolmes::EncodingDetector.detect_all 'test', 'UTF-8'
+    assert detected_list.is_a? Array
+
+    encoding_list = detected_list.map {|d| d[:encoding]}.sort
+    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
+  end
+
   test 'has a detect method' do
     @detector.respond_to? :detect
     detected = @detector.detect 'test'
     assert_equal 'ISO-8859-1', detected[:encoding]
   end
 
+  test 'has a detect method that accepts an encoding hint' do
+    @detector.respond_to? :detect
+    detected = @detector.detect 'test', 'UTF-8'
+    assert_equal 'ISO-8859-1', detected[:encoding]
+  end
+
   test 'has a detect_all method' do
     @detector.respond_to? :detect_all
     detected_list = @detector.detect_all 'test'
+    assert detected_list.is_a? Array
+
+    encoding_list = detected_list.map {|d| d[:encoding]}.sort
+    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
+  end
+
+  test 'has a detect_all method that accepts an encoding hint' do
+    @detector.respond_to? :detect_all
+    detected_list = @detector.detect_all 'test', 'UTF-8'
+    assert detected_list.is_a? Array
+
     encoding_list = detected_list.map {|d| d[:encoding]}.sort
     assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
   end

data/spec/string_method_spec.rb

@@ -10,6 +10,36 @@ describe String do
     assert_equal 'ISO-8859-1', detected[:encoding]
   end
 
+  test 'has a detect_encoding method that accepts an encoding hint' do
+    str = 'test'
+    str.respond_to? :detect_encoding
+
+    detected = str.detect_encoding 'UTF-8'
+    assert_equal 'ISO-8859-1', detected[:encoding]
+  end
+
+  test 'has a detect_encodings method' do
+    str = 'test'
+    str.respond_to? :detect_encodings
+
+    detected_list = str.detect_encodings
+    assert detected_list.is_a? Array
+
+    encoding_list = detected_list.map {|d| d[:encoding]}.sort
+    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
+  end
+
+  test 'has a detect_encodings method that accepts an encoding hint' do
+    str = 'test'
+    str.respond_to? :detect_encodings
+
+    detected_list = str.detect_encodings 'UTF-8'
+    assert detected_list.is_a? Array
+
+    encoding_list = detected_list.map {|d| d[:encoding]}.sort
+    assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
+  end
+
   if RUBY_VERSION =~ /1.9/
     test 'has a detect_encoding! method' do
       str = 'test'

metadata

@@ -1,58 +1,79 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: charlock_holmes
-version: !ruby/object:Gem::Version
-  version: 0.2.0
+version: !ruby/object:Gem::Version 
+  hash: 19
   prerelease: 
+  segments: 
+  - 0
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
-authors:
+authors: 
 - Brian Lopez
-- Vicent Martí
+- "Vicent Mart\xC3\xAD"
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-24 00:00:00.000000000 -07:00
+
+date: 2011-08-25 00:00:00 -07:00
 default_executable: 
-dependencies:
-- !ruby/object:Gem::Dependency
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: rake-compiler
-  requirement: &70293348071280 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 7
+        - 5
         version: 0.7.5
   type: :development
-  prerelease: false
-  version_requirements: *70293348071280
-- !ruby/object:Gem::Dependency
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
   name: rspec
-  requirement: &70293348070780 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 0
+        - 0
         version: 2.0.0
   type: :development
-  prerelease: false
-  version_requirements: *70293348070780
-- !ruby/object:Gem::Dependency
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
   name: chardet
-  requirement: &70293348070400 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
-  prerelease: false
-  version_requirements: *70293348070400
+  version_requirements: *id003
 description: 
 email: seniorlopez@gmail.com
 executables: []
-extensions:
+
+extensions: 
 - ext/charlock_holmes/extconf.rb
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
 - .rspec
 - Gemfile
@@ -81,31 +102,39 @@ files:
 has_rdoc: true
 homepage: http://github.com/brianmario/charlock_holmes
 licenses: []
+
 post_install_message: 
-rdoc_options:
+rdoc_options: 
 - --charset=UTF-8
-require_paths:
+require_paths: 
 - lib
 - ext
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
 rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Character encoding detection, brought to you by ICU
-test_files:
+test_files: 
 - spec/encoding_detector_spec.rb
 - spec/fixtures/AnsiGraph.psm1
 - spec/fixtures/TwigExtensionsDate.es.yml