RubyGems - guess_html_encoding - Versions diffs - 0.0.1 - Mend

guess_html_encoding 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (13) hide show

data/.gitignore +3 -0
data/.rvmrc +1 -0
data/Gemfile +4 -0
data/Gemfile.lock +24 -0
data/README +0 -0
data/Rakefile +6 -0
data/guess_html_encoding.gemspec +24 -0
data/lib/guess_html_encoding/version.rb +3 -0
data/lib/guess_html_encoding.rb +41 -0
data/spec/guess_html_encoding_spec.rb +60 -0
data/spec/spec.opts +4 -0
data/spec/spec_helper.rb +5 -0
metadata +71 -0

data/.gitignore ADDED Viewed

@@ -0,0 +1,3 @@
+.idea
+*.swp
+.DS_Store

data/.rvmrc ADDED Viewed

	@@ -0,0 +1 @@
1	+ rvm use 1.9.2@guess_html_encoding --create

data/Gemfile ADDED Viewed

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+# Specify your gem's dependencies in guess_html_encoding.gemspec
+gemspec

data/Gemfile.lock ADDED Viewed

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    guess_html_encoding (0.0.1)
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+PLATFORMS
+  ruby
+DEPENDENCIES
+  guess_html_encoding!
+  rspec

data/README ADDED Viewed

File without changes

data/Rakefile ADDED Viewed

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/guess_html_encoding.gemspec ADDED Viewed

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "guess_html_encoding/version"
+Gem::Specification.new do |s|
+  s.name        = "guess_html_encoding"
+  s.version     = GuessHtmlEncoding::VERSION
+  s.authors     = ["Andrew Cantino (Iteration Labs, LLC)"]
+  s.email       = ["andrew@iterationlabs.com"]
+  s.homepage    = "http://github.com/iterationlabs/guess_html_encoding"
+  s.summary     = %q{This gem helps guess the encoding of an HTML page.}
+  s.description = %q{This gem helps guess the encoding of an HTML page.}
+  s.rubyforge_project = "guess_html_encoding"
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/guess_html_encoding/version.rb ADDED Viewed

@@ -0,0 +1,3 @@
+module GuessHtmlEncoding
+  VERSION = "0.0.1"
+end

data/lib/guess_html_encoding.rb ADDED Viewed

@@ -0,0 +1,41 @@
+require "guess_html_encoding/version"
+module GuessHtmlEncoding
+  def self.guess(html, headers = nil)
+    html = html.dup.force_encoding("ASCII-8BIT")
+    headers = headers.dup.force_encoding("ASCII-8BIT") if headers
+    out = nil
+    (headers || "").split("\n").map {|i| i.split(":")}.each do |k,v|
+      if k =~ /Content-Type/i && v =~ /charset=([\w\d-]+);?/i
+        out = $1.upcase
+      end
+    end
+    if out.nil? || out.empty? || !Encoding.name_list.include?(out)
+      if html =~ /<meta[^>]*HTTP-EQUIV=["']Content-Type["'][^>]*content=["']([^'"]*)["']/i && $1 =~ /charset=([\w\d-]+);?/i
+        out = $1.upcase
+      end
+    end
+    if out
+      out = "UTF-8" if ["DEFAULT", "UTF8", "UNICODE"].include?(out.upcase)
+      out = "CP1251" if out.upcase == "CP-1251"
+      out = "ISO-8859-1" if ["LATIN1", "LATIN-1"].include?(out.upcase)
+      out = "Windows-1250" if ["WIN-1251", "WIN1251"].include?(out.upcase)
+    end
+    out
+  end
+  def self.encode(html, headers = nil)
+    encoding = guess(html, (headers || '').gsub(/[\r\n]+/, "\n"))
+    html.force_encoding(encoding ? encoding : "UTF-8")
+    if html.valid_encoding?
+      html
+    else
+      html.force_encoding('ASCII-8BIT').encode('UTF-8', :undef => :replace, :invalid => :replace)
+    end
+  end
+end

data/spec/guess_html_encoding_spec.rb ADDED Viewed

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+require 'spec_helper'
+describe "GuessHtmlEncoding" do
+  describe "#guess" do
+    it "prefers headers" do
+      guess = GuessHtmlEncoding.guess("<html><body><div>hi!</div></body></html>",
+                                      "Hello: world\nContent-Type: text/html; charset=utf-8\nFoo: bar")
+      guess.should == "UTF-8"
+    end
+    it "accepts meta tags" do
+      guess = GuessHtmlEncoding.guess('<html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"></head><body><div>hi!</div></body></html>')
+      guess.should == "UTF-8"
+    end
+    it "works okay when there is a semi-colon after the encoding with headers" do
+      guess = GuessHtmlEncoding.guess("<html><body><div>hi!</div></body></html>",
+                                      "Hello: world\nContent-Type: text/html; charset=utf-8;\nFoo: bar")
+      guess.should == "UTF-8"
+    end
+    it "works okay when there is a semi-colon after the encoding with meta-tags" do
+      guess = GuessHtmlEncoding.guess('<html><head><meta http-equiv="content-type" content="text/html; charset=utf-8;"></head><body><div>hi!</div></body></html>')
+      guess.should == "UTF-8"
+    end
+    it "converts UTF8 to UTF-8" do
+      guess = GuessHtmlEncoding.guess('<html><head><meta http-equiv="content-type" content="text/html; charset=utf8;"></head><body><div>hi!</div></body></html>')
+      guess.should == "UTF-8"
+    end
+    it "converts CP-1251 to CP1251" do
+      guess = GuessHtmlEncoding.guess('<html><head><meta http-equiv="content-type" content="text/html; charset=cp-1251;"></head><body><div>hi!</div></body></html>')
+      guess.should == "CP1251"
+    end
+    it "skips the header content type if it's invalid" do
+      guess = GuessHtmlEncoding.guess('<html><head><meta http-equiv="content-type" content="text/html; charset=utf8;"></head><body><div>hi!</div></body></html>',
+                                      "Hello: world\nContent-Type: text/html; charset=RU;\nFoo: bar")
+      guess.should == "UTF-8"
+    end
+  end
+  describe "#encode" do
+    it "should work on incorrectly encoded pages" do
+      data = "<html><head><meta http-equiv='content-type' content='text/html; charset=utf8;'></head><body><div>hi!\xc2</div></body></html>"
+      data.force_encoding("ASCII-8BIT")
+      data.should be_valid_encoding # everything is valid in binary
+      GuessHtmlEncoding.guess(data).should == "UTF-8" # because the page says so!
+      data.force_encoding("UTF-8").should_not be_valid_encoding # because of the bad byte sequence \xc2
+      encoded = GuessHtmlEncoding.encode(data)
+      encoded.encoding.to_s.should == "UTF-8"
+      encoded.should be_valid_encoding
+    end
+  end
+end

data/spec/spec.opts ADDED Viewed

@@ -0,0 +1,4 @@
+--colour
+--format s -c
+--loadby mtime
+--reverse

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'guess_html_encoding'
+RSpec.configure do |c|
+end

metadata ADDED Viewed

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: guess_html_encoding
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease:
+platform: ruby
+authors:
+- Andrew Cantino (Iteration Labs, LLC)
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2011-09-29 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70102956990840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70102956990840
+description: This gem helps guess the encoding of an HTML page.
+email:
+- andrew@iterationlabs.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- README
+- Rakefile
+- guess_html_encoding.gemspec
+- lib/guess_html_encoding.rb
+- lib/guess_html_encoding/version.rb
+- spec/guess_html_encoding_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+homepage: http://github.com/iterationlabs/guess_html_encoding
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: guess_html_encoding
+rubygems_version: 1.8.6
+signing_key:
+specification_version: 3
+summary: This gem helps guess the encoding of an HTML page.
+test_files:
+- spec/guess_html_encoding_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb