RubyGems - guess_html_encoding - Versions diffs - 0.0.1 → 0.0.2 - Mend

guess_html_encoding 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (7) hide show

data/.gitignore +1 -0
data/Gemfile.lock +1 -1
data/README +52 -0
data/lib/guess_html_encoding/version.rb +1 -1
data/lib/guess_html_encoding.rb +12 -5
data/spec/guess_html_encoding_spec.rb +25 -6
metadata +4 -4

data/.gitignore CHANGED Viewed

@@ -1,3 +1,4 @@
 .idea
 *.swp
+pkg
 .DS_Store

data/Gemfile.lock CHANGED Viewed

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    guess_html_encoding (0.0.1)
+    guess_html_encoding (0.0.2)
 GEM
   remote: http://rubygems.org/

data/README CHANGED Viewed

@@ -0,0 +1,52 @@
+GuessHtmlEncoding is a simple library to guess the HTML encodings in Ruby 1.9
+describe "GuessHtmlEncoding" do
+  describe "#guess" do
+    it "can use headers" do
+      guess = GuessHtmlEncoding.guess("<html><body><div>hi!</div></body></html>",
+                                      "Hello: world\nContent-Type: text/html; charset=LATIN1\nFoo: bar")
+      guess.should == "ISO-8859-1"
+    end
+    it "accepts headers as a hash as well" do
+      guess = GuessHtmlEncoding.guess("<html><body><div>hi!</div></body></html>",
+          {"Hello" => "world", "Content-Type" => "text/html; charset=LATIN1", "Foo" => "bar"})
+      guess.should == "ISO-8859-1"
+    end
+    it "accepts meta tags" do
+      guess = GuessHtmlEncoding.guess('<html><head><meta http-equiv="content-type" content="text/html; charset=LATIN1"></head><body><div>hi!</div></body></html>')
+      guess.should == "ISO-8859-1"
+    end
+    ...
+  end
+  describe "#encode" do
+    it "should work on correctly encoded pages" do
+      data = "<html><head><meta http-equiv='content-type' content='text/html; charset=utf8;'></head><body><div>hi!♥</div></body></html>"
+      data.force_encoding("ASCII-8BIT")
+      data.should be_valid_encoding # everything is valid in binary
+      GuessHtmlEncoding.guess(data).should == "UTF-8" # because the page says so!
+      data.force_encoding("UTF-8").should be_valid_encoding # because it really is utf-8
+      encoded = GuessHtmlEncoding.encode(data)
+      encoded.encoding.to_s.should == "UTF-8"
+      encoded.should be_valid_encoding
+    end
+    it "should work on incorrectly encoded pages" do
+      data = "<html><head><meta http-equiv='content-type' content='text/html; charset=utf8;'></head><body><div>hi!\xc2</div></body></html>"
+      data.force_encoding("ASCII-8BIT")
+      data.should be_valid_encoding # everything is valid in binary
+      GuessHtmlEncoding.guess(data).should == "UTF-8" # because the page says so!
+      data.force_encoding("UTF-8").should_not be_valid_encoding # because of the bad byte sequence \xc2 which is not valid UTF-8
+      encoded = GuessHtmlEncoding.encode(data)
+      encoded.encoding.to_s.should == "UTF-8"
+      encoded.should be_valid_encoding
+    end
+  end
+end

data/lib/guess_html_encoding/version.rb CHANGED Viewed

@@ -1,3 +1,3 @@
 module GuessHtmlEncoding
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/guess_html_encoding.rb CHANGED Viewed

@@ -1,15 +1,20 @@
 require "guess_html_encoding/version"
+# A small and simple library for guessing the encoding of HTML in Ruby 1.9.
 module GuessHtmlEncoding
+  # Guess the encoding of an HTML string, using HTTP headers if provided.  HTTP headers can be a string or a hash.
   def self.guess(html, headers = nil)
     html = html.dup.force_encoding("ASCII-8BIT")
-    headers = headers.dup.force_encoding("ASCII-8BIT") if headers
     out = nil
-    (headers || "").split("\n").map {|i| i.split(":")}.each do |k,v|
-      if k =~ /Content-Type/i && v =~ /charset=([\w\d-]+);?/i
-        out = $1.upcase
+    if headers
+      headers = headers.map {|k, v| "#{k}: #{v}" }.join("\n") if headers.is_a?(Hash)
+      headers = headers.dup.force_encoding("ASCII-8BIT")
+      headers.split("\n").map {|i| i.split(":")}.each do |k,v|
+        if k =~ /Content-Type/i && v =~ /charset=([\w\d-]+);?/i
+          out = $1.upcase
+          break
+        end
       end
     end
@@ -19,6 +24,7 @@ module GuessHtmlEncoding
       end
     end
+    # Translate encodings with other names.
     if out
       out = "UTF-8" if ["DEFAULT", "UTF8", "UNICODE"].include?(out.upcase)
       out = "CP1251" if out.upcase == "CP-1251"
@@ -29,6 +35,7 @@ module GuessHtmlEncoding
     out
   end
+  # Force an HTML string into a guessed encoding.
   def self.encode(html, headers = nil)
     encoding = guess(html, (headers || '').gsub(/[\r\n]+/, "\n"))
     html.force_encoding(encoding ? encoding : "UTF-8")

data/spec/guess_html_encoding_spec.rb CHANGED Viewed

@@ -3,15 +3,21 @@ require 'spec_helper'
 describe "GuessHtmlEncoding" do
   describe "#guess" do
-    it "prefers headers" do
+    it "can use headers" do
       guess = GuessHtmlEncoding.guess("<html><body><div>hi!</div></body></html>",
-                                      "Hello: world\nContent-Type: text/html; charset=utf-8\nFoo: bar")
-      guess.should == "UTF-8"
+                                      "Hello: world\nContent-Type: text/html; charset=LATIN1\nFoo: bar")
+      guess.should == "ISO-8859-1"
+    end
+    it "accepts headers as a hash as well" do
+      guess = GuessHtmlEncoding.guess("<html><body><div>hi!</div></body></html>",
+          {"Hello" => "world", "Content-Type" => "text/html; charset=LATIN1", "Foo" => "bar"})
+      guess.should == "ISO-8859-1"
     end
     it "accepts meta tags" do
-      guess = GuessHtmlEncoding.guess('<html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"></head><body><div>hi!</div></body></html>')
-      guess.should == "UTF-8"
+      guess = GuessHtmlEncoding.guess('<html><head><meta http-equiv="content-type" content="text/html; charset=LATIN1"></head><body><div>hi!</div></body></html>')
+      guess.should == "ISO-8859-1"
     end
     it "works okay when there is a semi-colon after the encoding with headers" do
@@ -44,13 +50,26 @@ describe "GuessHtmlEncoding" do
   end
   describe "#encode" do
+    it "should work on correctly encoded pages" do
+      data = "<html><head><meta http-equiv='content-type' content='text/html; charset=utf8;'></head><body><div>hi!♥</div></body></html>"
+      data.force_encoding("ASCII-8BIT")
+      data.should be_valid_encoding # everything is valid in binary
+      GuessHtmlEncoding.guess(data).should == "UTF-8" # because the page says so!
+      data.force_encoding("UTF-8").should be_valid_encoding # because it really is utf-8
+      encoded = GuessHtmlEncoding.encode(data)
+      encoded.encoding.to_s.should == "UTF-8"
+      encoded.should be_valid_encoding
+    end
     it "should work on incorrectly encoded pages" do
       data = "<html><head><meta http-equiv='content-type' content='text/html; charset=utf8;'></head><body><div>hi!\xc2</div></body></html>"
       data.force_encoding("ASCII-8BIT")
       data.should be_valid_encoding # everything is valid in binary
       GuessHtmlEncoding.guess(data).should == "UTF-8" # because the page says so!
-      data.force_encoding("UTF-8").should_not be_valid_encoding # because of the bad byte sequence \xc2
+      data.force_encoding("UTF-8").should_not be_valid_encoding # because of the bad byte sequence \xc2 which is not valid UTF-8
       encoded = GuessHtmlEncoding.encode(data)
       encoded.encoding.to_s.should == "UTF-8"

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: guess_html_encoding
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease:
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2011-09-29 00:00:00.000000000Z
+date: 2011-10-26 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70102956990840 !ruby/object:Gem::Requirement
+  requirement: &70345601879200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70102956990840
+  version_requirements: *70345601879200
 description: This gem helps guess the encoding of an HTML page.
 email:
 - andrew@iterationlabs.com