charlock_holmes 0.3.0 → 0.4.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    charlock_holmes (0.2.0)
+    charlock_holmes (0.3.0)
 
 GEM
   remote: http://rubygems.org/

data/ext/charlock_holmes/charlock_holmes.c

@@ -8,6 +8,15 @@
 static VALUE rb_mCharlockHolmes;
 static VALUE rb_cEncodingDetector;
 
+static VALUE charlock_new_str(const char *str, size_t len)
+{
+#ifdef HAVE_RUBY_ENCODING_H
+	return rb_external_str_new_with_enc(str, len, rb_utf8_encoding());
+#else
+	return rb_str_new(str, len);
+#endif
+}
+
 static VALUE charlock_new_str2(const char *str)
 {
 #ifdef HAVE_RUBY_ENCODING_H
@@ -89,7 +98,7 @@ static VALUE rb_encdec_detect(int argc, VALUE *argv, VALUE self)
  *            be used as an additional hint to the charset detector
  *
  * Returns: an Array with zero or more Hashes,
- *	each one of them with with :encoding, :language and :confidence
+ *          each one of them with with :encoding, :language and :confidence
  */
 static VALUE rb_encdec_detect_all(int argc, VALUE *argv, VALUE self)
 {
@@ -124,6 +133,90 @@ static VALUE rb_encdec_detect_all(int argc, VALUE *argv, VALUE self)
 	return rb_ret;
 }
 
+/*
+ * call-seq: EncodingDetector#strip_tags?
+ *
+ * Returns whether or not the strip_tags flag is set on this detector
+ *
+ * Returns: Boolean
+ */
+static VALUE rb_get_strip_tags(VALUE self)
+{
+	UCharsetDetector *csd;
+	UBool val;
+	VALUE rb_val;
+
+	Data_Get_Struct(self, UCharsetDetector, csd);
+
+	val = ucsdet_isInputFilterEnabled(csd);
+
+	rb_val = val == 1 ? Qtrue : Qfalse;
+
+	return rb_val;
+}
+
+/*
+ * call-seq: EncodingDetector#strip_tags = true
+ *
+ * Enable or disable the stripping of HTML/XML tags from the input before
+ * attempting any detection
+ *
+ * Returns: Boolean, the value passed
+ */
+static VALUE rb_set_strip_tags(VALUE self, VALUE rb_val)
+{
+	UCharsetDetector *csd;
+	UBool val;
+
+	Data_Get_Struct(self, UCharsetDetector, csd);
+
+	val = rb_val == Qtrue ? 1 : 0;
+
+	ucsdet_enableInputFilter(csd, val);
+
+	return rb_val;
+}
+
+/*
+ * call-seq: detectable_encodings = EncodingDetector.supported_encodings
+ *
+ * The list of detectable encodings supported by this library
+ *
+ * Returns: an Array of Strings
+ */
+static VALUE rb_get_supported_encodings(VALUE klass)
+{
+	UCharsetDetector *csd;
+	UErrorCode status = U_ZERO_ERROR;
+	UEnumeration *encoding_list;
+	VALUE rb_encoding_list;
+	int32_t enc_count;
+	int32_t i;
+	const char *enc_name;
+	int32_t enc_name_len;
+
+	csd = ucsdet_open(&status);
+
+	rb_encoding_list = rb_iv_get(klass, "encoding_list");
+
+	// lazily populate the list
+	if (NIL_P(rb_encoding_list)) {
+		encoding_list = ucsdet_getAllDetectableCharsets(csd, &status);
+		rb_encoding_list = rb_ary_new();
+		enc_count = uenum_count(encoding_list, &status);
+
+		for(i=0; i < enc_count; i++) {
+			enc_name = uenum_next(encoding_list, &enc_name_len, &status);
+			rb_ary_push(rb_encoding_list, charlock_new_str(enc_name, enc_name_len));
+		}
+
+		rb_iv_set(klass, "encoding_list", rb_encoding_list);
+	}
+
+	ucsdet_close(csd);
+
+	return rb_encoding_list;
+}
 
 static void rb_encdec__free(void *csd)
 {
@@ -145,4 +238,8 @@ void Init_charlock_holmes()
 	rb_define_alloc_func(rb_cEncodingDetector, rb_encdec__alloc);
 	rb_define_method(rb_cEncodingDetector, "detect", rb_encdec_detect, -1);
 	rb_define_method(rb_cEncodingDetector, "detect_all", rb_encdec_detect_all, -1);
+	rb_define_method(rb_cEncodingDetector, "strip_tags", rb_get_strip_tags, 0);
+	rb_define_method(rb_cEncodingDetector, "strip_tags=", rb_set_strip_tags, 1);
+
+	rb_define_singleton_method(rb_cEncodingDetector, "supported_encodings", rb_get_supported_encodings, 0);
 }

data/lib/charlock_holmes/encoding_detector.rb

@@ -1,5 +1,7 @@
 module CharlockHolmes
   class EncodingDetector
+    alias :strip_tags? :strip_tags
+
     # Attempt to detect the encoding of this string
     #
     # NOTE: This will create a new CharlockHolmes::EncodingDetector instance on every call

data/lib/charlock_holmes/version.rb

@@ -1,3 +1,3 @@
 module CharlockHolmes
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/spec/encoding_detector_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'spec_helper'
 
 describe CharlockHolmes::EncodingDetector do
@@ -65,6 +67,29 @@ describe CharlockHolmes::EncodingDetector do
     assert_equal ['ISO-8859-1', 'ISO-8859-2', 'UTF-8'], encoding_list
   end
 
+  test 'has a strip_tags flag' do
+    detector = CharlockHolmes::EncodingDetector.new
+    detector.strip_tags = true
+    assert detector.strip_tags
+
+    detection = detector.detect "<div ascii_attribute='some more ascii'>λ, λ, λ</div>"
+    assert_equal 'UTF-8', detection[:encoding]
+
+    detector.strip_tags = false
+    assert !detector.strip_tags
+
+    detection = detector.detect "<div ascii_attribute='some more ascii'>λ, λ, λ</div>"
+    assert_equal 'UTF-8', detection[:encoding]
+  end
+
+  test 'has a list of supported encodings' do
+    CharlockHolmes::EncodingDetector.respond_to? :supported_encodings
+    supported_encodings = CharlockHolmes::EncodingDetector.supported_encodings
+
+    assert supported_encodings.is_a?(Array)
+    assert supported_encodings.include? 'UTF-8'
+  end
+
   context 'encoding detection' do
     MAPPING = [
       ['repl2.cljs',                'ISO-8859-1'],

metadata

@@ -1,79 +1,58 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: charlock_holmes
-version: !ruby/object:Gem::Version 
-  hash: 19
+version: !ruby/object:Gem::Version
+  version: 0.4.0
   prerelease: 
-  segments: 
-  - 0
-  - 3
-  - 0
-  version: 0.3.0
 platform: ruby
-authors: 
+authors:
 - Brian Lopez
-- "Vicent Mart\xC3\xAD"
+- Vicent Martí
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-08-25 00:00:00 -07:00
+date: 2011-08-25 00:00:00.000000000 -07:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rake-compiler
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70365951363440 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 9
-        segments: 
-        - 0
-        - 7
-        - 5
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.7.5
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: rspec
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *70365951363440
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70365951362940 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 2
-        - 0
-        - 0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 2.0.0
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: chardet
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *70365951362940
+- !ruby/object:Gem::Dependency
+  name: chardet
+  requirement: &70365951394040 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id003
+  prerelease: false
+  version_requirements: *70365951394040
 description: 
 email: seniorlopez@gmail.com
 executables: []
-
-extensions: 
+extensions:
 - ext/charlock_holmes/extconf.rb
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - .rspec
 - Gemfile
@@ -102,39 +81,31 @@ files:
 has_rdoc: true
 homepage: http://github.com/brianmario/charlock_holmes
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --charset=UTF-8
-require_paths: 
+require_paths:
 - lib
 - ext
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Character encoding detection, brought to you by ICU
-test_files: 
+test_files:
 - spec/encoding_detector_spec.rb
 - spec/fixtures/AnsiGraph.psm1
 - spec/fixtures/TwigExtensionsDate.es.yml