charazard 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 34b9d5367e647921915519d5a3a252f683ddf91b
+  data.tar.gz: b762b59f2e9ddeeefb4ade4afe6511a6756d1074
+SHA512:
+  metadata.gz: 887e42a951e9fcb07146bcf9f7adc7dbde56f706dd6d964d32d345112bc27508a38d6210430218e2165ca785d07f1c5e37d840e74198938dc3a35c58f8b19a81
+  data.tar.gz: dd537a7e4d2ae0d675f06731a3aac65e729a0b28f341d6c4adcc8ec60073ab312aefa605722b03b3b093d76e5470af79bf430dc3d38b5e683a24390399334e1e

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+pkg
+.bundle

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Jason Weathered
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,64 @@
+# Charazard
+
+Cleans up bad character encodings with liberal application of fire.
+
+## Usage
+
+### Converting Windows-1252 (ISO 8859-1) to UTF-8
+
+CSV files saved by Excel on Windows are by default encoded in Windows-1252 (which is close to but not quite ISO 8859-1 or ISO Latin 1).
+`Charazard.fix_invalid_unicode_literals` can be used to convert these characters into valid UTF-8 without breaking existing UTF-8 strings.
+
+```ruby
+Charazard.fix_invalid_unicode_literals("\x93Smart quotes\x94 \xC3\x9Cber Unicode")
+  # => "“Smart quotes” Über Unicode"
+```
+
+`Charazard.fix_invalid_unicode_literals` can be used in combination with
+[`filter_io`](https://github.com/jasoncodes/filter_io) to filter CSV streams.
+Here’s an example that handles UTF-8/ISO-8859-1 with mixed line endings:
+
+``` ruby
+require 'filter_io'
+require 'charazard'
+require 'csv'
+
+File.open(filename, external_encoding: 'UTF-8') do |io|
+  io = FilterIO.new(io) do |data, state|
+    # fix invalid UTF-8 literals
+    data = Charazard.fix_invalid_unicode_literals(data)
+
+    # grab another chunk if the last character is a delimiter
+    raise FilterIO::NeedMoreData if data =~ /[\r\n]\z/ && !state.eof?
+    # normalise line endings to LF
+    data = data.gsub /\r\n|\r|\n/, "\n"
+
+    data
+  end
+
+  CSV.parse(io, row_sep: "\n") do |row|
+    p row
+  end
+end
+```
+
+### Converting HTML to plain text
+
+Feeds (RSS, product lists, etc.) often contain HTML which you may want to sanitize into plain text.
+Charazard recognises basic formatting such as paragraphs and lists.
+
+```html
+<p>First sentence.</p><p>Second sentenence.</p>
+<ul><li>foo</li><li>bar</li></ul>
+```
+
+```ruby
+text = Charazard.html_to_plain(html)
+```
+
+```markdown
+First sentence.
+Second sentenence.
+* foo
+* bar
+```

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+task :default => :test
+
+desc 'Run the tests'
+Rake::TestTask.new do |t|
+  t.libs.push 'lib'
+  t.libs.push 'test'
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end
+
+desc 'Open a Pry console with environment'
+task :console do
+  exec "pry -Ilib -rcharazard"
+end

data/charazard.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'charazard/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'charazard'
+  spec.version = Charazard::VERSION
+  spec.authors = ['Jason Weathered']
+  spec.email = ['jason@jasoncodes.com']
+  spec.summary = %q{Cleans up bad character encodings with liberal application of fire.}
+  spec.homepage = 'https://github.com/jasoncodes/charazard'
+  spec.license = 'MIT'
+
+  spec.files = `git ls-files -z`.split("\x0")
+  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'nokogiri', '~> 1.5'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'pry'
+end

data/lib/charazard/version.rb

@@ -0,0 +1,3 @@
+module Charazard
+  VERSION = '0.0.1'
+end

data/lib/charazard.rb

@@ -0,0 +1,107 @@
+# coding: utf-8
+
+require 'charazard/version'
+require 'nokogiri'
+
+module Charazard
+  extend self
+
+  CP1252_TABLE = {
+    128 => 0x20AC, 130 => 0x201A, 131 => 0x0192, 132 => 0x201E, 133 => 0x2026, 134 => 0x2020, 135 => 0x2021,
+    136 => 0x02C6, 137 => 0x2030, 138 => 0x0160, 139 => 0x2039, 140 => 0x0152, 142 => 0x017D, 145 => 0x2018,
+    146 => 0x2019, 147 => 0x201C, 148 => 0x201D, 149 => 0x2022, 150 => 0x2013, 151 => 0x2014, 152 => 0x02DC,
+    153 => 0x2122, 154 => 0x0161, 155 => 0x203A, 156 => 0x0153, 158 => 0x017E, 159 => 0x0178
+  }
+
+  UTF8_REGEX_PREFIX = /\A(
+         [\x09\x0A\x0D\x20-\x7E]            # ASCII
+       | [\xC2-\xDF][\x80-\xBF]             # non-overlong 2-byte
+       |  \xE0[\xA0-\xBF][\x80-\xBF]        # excluding overlongs
+       | [\xE1-\xEC\xEE][\x80-\xBF]{2}      # straight 3-byte
+       |  \xEF[\x80-\xBE]{2}                #
+       |  \xEF\xBF[\x80-\xBD]               # excluding U+fffe and U+ffff
+       |  \xED[\x80-\x9F][\x80-\xBF]        # excluding surrogates
+       |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
+       | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
+       |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
+     )*/nx;
+
+  def fix_invalid_unicode_literals(str)
+    input = str.dup.force_encoding('ASCII-8BIT')
+    input.gsub!(/\xef\xbb\xbf/n, '') # byte order marker
+    input.gsub!(/\xef\xbf\xbd/n, '') # replacement character
+    if input =~ /[\x80-\xFF]/n
+      output = input.slice! 0, 0
+      until input.empty?
+        input =~ UTF8_REGEX_PREFIX or raise "UTF8 match failed"
+        input = $'
+        output << $&
+        unless input.empty?
+          byte = input.slice!(0,1).ord
+          char = if CP1252_TABLE[byte]
+            [CP1252_TABLE[byte]].pack 'U'
+          else
+            [byte].pack 'U'
+          end
+          output << char.force_encoding('ASCII-8BIT')
+        end
+      end
+      output.force_encoding('UTF-8')
+    else
+      input.force_encoding('UTF-8')
+    end
+  end
+
+  def strip_dom(str, keep_newline = false)
+    str = str.gsub(/\v/, '') # remove vertical tabs
+    str = str.gsub(/(?:\s|\xC2\xA0|\xEF\xBF\xBD)+/) do |match|
+      if keep_newline && match =~ /(?:\r|\n)/
+        "\n"
+      else
+        " "
+      end
+    end
+    str = str.gsub(/[\x00-\x09\x0b-\x1f]/, '') # remove all chars < 0x20 except 0x10 (LF)
+    str.strip
+  end
+
+  def fix_cp1252_entities(str)
+    if str =~ /&#1[345][0-9];/
+      CP1252_TABLE.each do |from,to|
+        str = str.gsub "&##{from};", "&##{to};"
+      end
+    end
+    str
+  end
+
+  def html_to_plain(str, keep_newline = true)
+    str = fix_invalid_unicode_literals(str)
+
+    if str.include? '<' or str.include? '&'
+      str = fix_cp1252_entities(strip_dom(str))
+      str.gsub!(/-<-/, '-&lt;-')
+      doc = Nokogiri::HTML.fragment(str)
+      doc.search('style,script').each(&:remove)
+      doc.search('br').each do |br|
+        br.replace(Nokogiri::XML::Text.new("\n", doc))
+      end
+      doc.search('p,div').each do |p|
+        p.add_next_sibling(Nokogiri::XML::Text.new("\n", doc))
+        p.add_previous_sibling(Nokogiri::XML::Text.new("\n", doc))
+      end
+      doc.search('ul,ol').each do |list|
+        is_ordered = list.node_name.downcase == 'ol'
+        start = list[:start] =~ /\A(\d+)\z/ ? $1.to_i : 1
+        list.add_next_sibling Nokogiri::XML::Text.new("\n", doc)
+        list.add_previous_sibling Nokogiri::XML::Text.new("\n", doc)
+        list.css('li').each_with_index do |item,index|
+          item_prefix_text = is_ordered ? "\n#{start+index}. " : "\n* "
+          item.add_previous_sibling Nokogiri::XML::Text.new(item_prefix_text, doc)
+        end
+      end
+      str = doc.inner_text
+    end
+
+    return strip_dom(str, keep_newline)
+  end
+end

data/test/charazard_test.rb

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+
+require 'test_helper'
+
+describe Charazard do
+  describe 'fix_invalid_unicode_literals' do
+    test "Test ISO-8859-1 literal conversion is idempotent" do
+      assert_equal Charazard.html_to_plain("™ © ®"), Charazard.fix_invalid_unicode_literals(Charazard.fix_invalid_unicode_literals("\231 \251 \256"))
+    end
+
+    test "ISO-8859-1 characters within words" do
+      assert_equal "M\xc3\xa9nage \xc3\xa0 Trois for", Charazard.fix_invalid_unicode_literals("M\xe9nage \xe0 Trois for")
+    end
+
+    test "should fix literals in ASCII-8BIT without breaking correct UTF-8" do
+      input = "r\xc3\xa9sum\xe9"
+      input.force_encoding 'ASCII-8BIT'
+      expected = "r\xc3\xa9sum\xc3\xa9"
+      output = Charazard.fix_invalid_unicode_literals(input)
+      assert_equal expected, output
+    end
+
+    test "replacement character (U+FFFD, UTF-8 EF BF BD) as invalid literal" do
+      assert_equal "foo bar", Charazard.fix_invalid_unicode_literals("foo \xef\xbf\xbdbar")
+    end
+
+    test "byte order marker (U+FEFF, UTF-8 EF BB BF) as invalid literal" do
+      assert_equal "foo bar", Charazard.fix_invalid_unicode_literals("foo \xef\xbb\xbfbar")
+    end
+  end
+
+  describe 'strip_dom' do
+    test "whitespace normalisation removing newlines" do
+      assert_equal "foo bar", Charazard.strip_dom("  foo \n\t\tbar  \n  ")
+    end
+
+    test "whitespace normalisation keeping newlines" do
+      assert_equal "foo\nbar", Charazard.strip_dom("  foo \n\t\tbar  \n  ", true)
+    end
+
+    test "newline preservation with multiple newlines together" do
+      assert_equal "foo\nbar\nbaz", Charazard.strip_dom("foo\n\n\n\nbar\t\n\t\n\tbaz", true)
+    end
+
+    test "non-breaking space (UTF-8 C2 A0)" do
+      assert_equal "foo bar", Charazard.strip_dom("foo\xc2\xa0bar")
+    end
+
+    test "replacement character (U+FFFD, UTF-8 EF BF BD)" do
+      assert_equal "foo bar", Charazard.strip_dom("foo\xef\xbf\xbdbar")
+    end
+
+    test "CR in newline removal" do
+      assert_equal "foo bar", Charazard.strip_dom("foo\rbar", false)
+    end
+
+    test "CR in newline preservation" do
+      assert_equal "foo\nbar", Charazard.strip_dom("foo\rbar", true)
+    end
+
+    test "invalid Unicode code points below 0x20" do
+      assert_equal "foobar", Charazard.strip_dom("foo\006bar")
+      assert_equal "foobar\nbaz", Charazard.strip_dom("foo\006bar\n  baz", true)
+      assert_equal "shes", Charazard.strip_dom("\x73\x68\x65\x1a\x73")
+    end
+  end
+
+  describe 'html_to_plain' do
+    test "html basic entities" do
+      assert_equal "foo & \"bar\" <baz>", Charazard.html_to_plain("foo &amp; &quot;bar\" &lt;baz&gt;")
+      assert_equal "foo\xc2\xae bar \xe2\x82\xac123!", Charazard.html_to_plain("foo&reg; bar &euro;12&#51;&#33;")
+    end
+
+    test "html basic pass through" do
+      assert_equal "foo bar", Charazard.html_to_plain("foo bar")
+    end
+
+    test "html basic whitespace" do
+      assert_equal "foo bar baz. Hello world. 1 2 3.", Charazard.html_to_plain("foo&nbsp;bar baz.&nbsp; Hello world.  1 2 3.")
+    end
+
+    test "html basic elements" do
+      assert_equal "This is a test.", Charazard.html_to_plain("<strong>This</strong> <em>is <span style='text-decoration: underline'>a</span></em> <a href='http://www.example.com/' title=\"Testing\">test</a>.")
+    end
+
+    test "html block entities" do
+      assert_equal "Foo\nBar\nabc\ndef\nghi", Charazard.html_to_plain("<p>Foo</p><p>Bar</p><p>abc<p>def<br />ghi")
+    end
+
+    test "invalid Unicode code points in numeric entities from CP1252" do
+      assert_equal "that\xe2\x80\x99s \xe2\x80\x93 a test", Charazard.html_to_plain("that&#146;s &#150; a test")
+    end
+
+    test "Test common ISO-8859-1 literals" do
+      assert_equal Charazard.html_to_plain("Foo&reg; Bar&trade; &copy;2010"), Charazard.html_to_plain("Foo\256 Bar\231 \2512010")
+    end
+
+    test "ISO-8859-1 capital A grave accent literal" do
+      assert_equal "\xc3\x82", Charazard.html_to_plain(Charazard.html_to_plain("\xc2"))
+    end
+
+    test "Line tabulation character U+000B in HTML should be stripped" do
+      assert_equal 'foobar', Charazard.html_to_plain("foo\x0bbar")
+    end
+
+    test "Newline U+000A in HTML should be preserved" do
+      assert_equal "foo\nbar", Charazard.html_to_plain("foo\nbar")
+    end
+
+    test "Tab U+0009 in HTML should be normalised to space" do
+      assert_equal "foo bar", Charazard.html_to_plain("foo\tbar")
+    end
+
+    test "Combination of different conversion cases in HTML to plain" do
+      assert_equal "A\nB\xc3\x82CD", Charazard.html_to_plain(Charazard.html_to_plain("A\nB\xc2C\x0bD"))
+    end
+
+    test "stylesheet and script blocks should be stripped" do
+      html = <<-HTML
+        foo
+        <script>alert('hi');</script><strong>bar</strong>
+        <style type="text/css">
+        body
+        {
+          font-face: Helvetica;
+        }
+        </style>
+        baz
+      HTML
+      plain = "foo bar baz"
+      assert_equal plain, Charazard.html_to_plain(html)
+    end
+
+    test "unordered lists should format as lines" do
+      html = "foo<ul><li>one</li><li>two</li><li>three</li></ul>bar"
+      plain = "foo\n* one\n* two\n* three\nbar"
+      assert_equal plain, Charazard.html_to_plain(html)
+    end
+
+    test "ordered lists should format as lines" do
+      html = "foo<ol><li>one</li><li>two</li><li>three</li></ol>bar"
+      plain = "foo\n1. one\n2. two\n3. three\nbar"
+      assert_equal plain, Charazard.html_to_plain(html)
+    end
+
+    test "ordered list with start offset" do
+      html = "<ol start=3><li>three</li><li>four</li></ol>"
+      plain = "3. three\n4. four"
+      assert_equal plain, Charazard.html_to_plain(html)
+    end
+
+    test "should be converted to UTF-8" do
+      input = "Foo\nBar\x99".force_encoding('ASCII-8BIT').split("\n").to_a
+      expected = ["Foo", "Bar\xE2\x84\xA2"]
+      actual = input.map { |str| Charazard.html_to_plain(str) }
+      assert_equal expected, actual
+      expected.zip(actual).each do |expected_line, actual_line|
+        assert_equal expected_line.encoding, actual_line.encoding
+      end
+    end
+
+    test "URL containing hyphen-less-than-hyphen passes through" do
+      str = "http://example.com/age-<-5-years"
+      assert_equal str, Charazard.html_to_plain(str)
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,8 @@
+require 'minitest/spec'
+require 'minitest/autorun'
+
+MiniTest::Spec::DSL.class_eval do
+  alias_method :test, :it
+end
+
+require 'charazard'

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: charazard
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jason Weathered
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- jason@jasoncodes.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.markdown
+- Rakefile
+- charazard.gemspec
+- lib/charazard.rb
+- lib/charazard/version.rb
+- test/charazard_test.rb
+- test/test_helper.rb
+homepage: https://github.com/jasoncodes/charazard
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Cleans up bad character encodings with liberal application of fire.
+test_files:
+- test/charazard_test.rb
+- test/test_helper.rb