html-pipeline-no-charlock 0.0.6

data/test/html/pipeline/markdown_filter_test.rb

@@ -0,0 +1,101 @@
+require "test_helper"
+
+MarkdownFilter = HTML::Pipeline::MarkdownFilter
+
+class HTML::Pipeline::MarkdownFilterTest < Test::Unit::TestCase
+  def setup
+    @haiku =
+      "Pointing at the moon\n" +
+      "Reminded of simple things\n" +
+      "Moments matter most"
+    @links =
+      "See http://example.org/ for more info"
+    @code =
+      "```\n" +
+      "def hello()" +
+      "  'world'" +
+      "end" +
+      "```"
+  end
+
+  def test_fails_when_given_a_documentfragment
+    body = "<p>heyo</p>"
+    doc  = HTML::Pipeline.parse(body)
+    assert_raise(TypeError) { MarkdownFilter.call(doc, {}) }
+  end
+
+  def test_gfm_enabled_by_default
+    doc = MarkdownFilter.to_document(@haiku, {})
+    assert doc.kind_of?(HTML::Pipeline::DocumentFragment)
+    assert_equal 2, doc.search('br').size
+  end
+
+  def test_disabling_gfm
+    doc = MarkdownFilter.to_document(@haiku, :gfm => false)
+    assert doc.kind_of?(HTML::Pipeline::DocumentFragment)
+    assert_equal 0, doc.search('br').size
+  end
+
+  def test_fenced_code_blocks
+    doc = MarkdownFilter.to_document(@code)
+    assert doc.kind_of?(HTML::Pipeline::DocumentFragment)
+    assert_equal 1, doc.search('pre').size
+  end
+
+  def test_fenced_code_blocks_with_language
+    doc = MarkdownFilter.to_document(@code.sub("```", "``` ruby"))
+    assert doc.kind_of?(HTML::Pipeline::DocumentFragment)
+    assert_equal 1, doc.search('pre').size
+    assert_equal 'ruby', doc.search('pre').first['lang']
+  end
+end
+
+class GFMTest < Test::Unit::TestCase
+  def gfm(text)
+    MarkdownFilter.call(text, :gfm => true)
+  end
+
+  def test_not_touch_single_underscores_inside_words
+    assert_equal "<p>foo_bar</p>",
+                 gfm("foo_bar")
+  end
+
+  def test_not_touch_underscores_in_code_blocks
+    assert_equal "<pre><code>foo_bar_baz\n</code></pre>",
+                 gfm("    foo_bar_baz")
+  end
+
+  def test_not_touch_underscores_in_pre_blocks
+    assert_equal "<pre>\nfoo_bar_baz\n</pre>",
+                 gfm("<pre>\nfoo_bar_baz\n</pre>")
+  end
+
+  def test_not_touch_two_or_more_underscores_inside_words
+    assert_equal "<p>foo_bar_baz</p>",
+                 gfm("foo_bar_baz")
+  end
+
+  def test_turn_newlines_into_br_tags_in_simple_cases
+    assert_equal "<p>foo<br>\nbar</p>",
+                 gfm("foo\nbar")
+  end
+
+  def test_convert_newlines_in_all_groups
+    assert_equal "<p>apple<br>\npear<br>\norange</p>\n\n" +
+                 "<p>ruby<br>\npython<br>\nerlang</p>",
+                 gfm("apple\npear\norange\n\nruby\npython\nerlang")
+  end
+
+  def test_convert_newlines_in_even_long_groups
+    assert_equal "<p>apple<br>\npear<br>\norange<br>\nbanana</p>\n\n" +
+                 "<p>ruby<br>\npython<br>\nerlang</p>",
+                 gfm("apple\npear\norange\nbanana\n\nruby\npython\nerlang")
+  end
+
+  def test_not_convert_newlines_in_lists
+    assert_equal "<h1>foo</h1>\n\n<h1>bar</h1>",
+                 gfm("# foo\n# bar")
+    assert_equal "<ul>\n<li>foo</li>\n<li>bar</li>\n</ul>",
+                 gfm("* foo\n* bar")
+  end
+end

data/test/html/pipeline/mention_filter_test.rb

@@ -0,0 +1,158 @@
+require "test_helper"
+
+class HTML::Pipeline::MentionFilterTest < Test::Unit::TestCase
+  def filter(html, base_url='/', info_url=nil)
+    HTML::Pipeline::MentionFilter.call(html, :base_url => base_url, :info_url => info_url)
+  end
+
+  def test_filtering_a_documentfragment
+    body = "<p>@kneath: check it out.</p>"
+    doc  = Nokogiri::HTML::DocumentFragment.parse(body)
+
+    res  = filter(doc, '/')
+    assert_same doc, res
+
+    link = "<a href=\"/kneath\" class=\"user-mention\">@kneath</a>"
+    assert_equal "<p>#{link}: check it out.</p>",
+      res.to_html
+  end
+
+  def test_filtering_plain_text
+    body = "<p>@kneath: check it out.</p>"
+    res  = filter(body, '/')
+
+    link = "<a href=\"/kneath\" class=\"user-mention\">@kneath</a>"
+    assert_equal "<p>#{link}: check it out.</p>",
+      res.to_html
+  end
+
+  def test_not_replacing_mentions_in_pre_tags
+    body = "<pre>@kneath: okay</pre>"
+    assert_equal body, filter(body).to_html
+  end
+
+  def test_not_replacing_mentions_in_code_tags
+    body = "<p><code>@kneath:</code> okay</p>"
+    assert_equal body, filter(body).to_html
+  end
+
+  def test_not_replacing_mentions_in_links
+    body = "<p><a>@kneath</a> okay</p>"
+    assert_equal body, filter(body).to_html
+  end
+
+  def test_entity_encoding_and_whatnot
+    body = "<p>@&#x6b;neath what's up</p>"
+    link = "<a href=\"/kneath\" class=\"user-mention\">@kneath</a>"
+    assert_equal "<p>#{link} what's up</p>", filter(body, '/').to_html
+  end
+
+  def test_html_injection
+    body = "<p>@kneath &lt;script>alert(0)&lt;/script></p>"
+    link = "<a href=\"/kneath\" class=\"user-mention\">@kneath</a>"
+    assert_equal "<p>#{link} &lt;script&gt;alert(0)&lt;/script&gt;</p>",
+      filter(body, '/').to_html
+  end
+
+  def test_links_to_nothing_when_no_info_url_given
+    body = "<p>How do I @mention someone?</p>"
+    assert_equal "<p>How do I @mention someone?</p>",
+      filter(body, '/').to_html
+  end
+
+  def test_links_to_more_info_when_info_url_given
+    body = "<p>How do I @mention someone?</p>"
+    link = "<a href=\"https://github.com/blog/821\" class=\"user-mention\">@mention</a>"
+    assert_equal "<p>How do I #{link} someone?</p>",
+      filter(body, '/', 'https://github.com/blog/821').to_html
+  end
+
+  MarkdownPipeline =
+    HTML::Pipeline.new [
+      HTML::Pipeline::MarkdownFilter,
+      HTML::Pipeline::MentionFilter
+    ]
+
+  def mentioned_usernames
+    result = {}
+    MarkdownPipeline.call(@body, {}, result)
+    html = result[:output].to_html
+    users = html.scan(/user-mention">@(.+?)</)
+    users ? users.flatten.uniq : []
+  end
+
+  def test_matches_usernames_in_body
+    @body = "@test how are you?"
+    assert_equal %w[test], mentioned_usernames
+  end
+
+  def test_matches_usernames_with_dashes
+    @body = "hi @some-user"
+    assert_equal %w[some-user], mentioned_usernames
+  end
+
+  def test_matches_usernames_followed_by_a_single_dot
+    @body = "okay @some-user."
+    assert_equal %w[some-user], mentioned_usernames
+  end
+
+  def test_matches_usernames_followed_by_multiple_dots
+    @body = "okay @some-user..."
+    assert_equal %w[some-user], mentioned_usernames
+  end
+
+  def test_does_not_match_email_addresses
+    @body = "aman@tmm1.net"
+    assert_equal [], mentioned_usernames
+  end
+
+  def test_does_not_match_domain_name_looking_things
+    @body = "we need a @github.com email"
+    assert_equal [], mentioned_usernames
+  end
+
+  def test_does_not_match_organization_team_mentions
+    @body = "we need to @github/enterprise know"
+    assert_equal [], mentioned_usernames
+  end
+
+  def test_matches_colon_suffixed_names
+    @body = "@tmm1: what do you think?"
+    assert_equal %w[tmm1], mentioned_usernames
+  end
+
+  def test_matches_list_of_names
+    @body = "@defunkt @atmos @kneath"
+    assert_equal %w[defunkt atmos kneath], mentioned_usernames
+  end
+
+  def test_matches_list_of_names_with_commas
+    @body = "/cc @defunkt, @atmos, @kneath"
+    assert_equal %w[defunkt atmos kneath], mentioned_usernames
+  end
+
+  def test_matches_inside_brackets
+    @body = "(@mislav) and [@rtomayko]"
+    assert_equal %w[mislav rtomayko], mentioned_usernames
+  end
+
+  def test_doesnt_ignore_invalid_users
+    @body = "@defunkt @mojombo and @somedude"
+    assert_equal ['defunkt', 'mojombo', 'somedude'], mentioned_usernames
+  end
+
+  def test_returns_distinct_set
+    @body = "/cc @defunkt, @atmos, @kneath, @defunkt, @defunkt"
+    assert_equal %w[defunkt atmos kneath], mentioned_usernames
+  end
+
+  def test_does_not_match_inline_code_block_with_multiple_code_blocks
+    @body = "something\n\n`/cc @defunkt @atmos @kneath` `/cc @atmos/atmos`"
+    assert_equal %w[], mentioned_usernames
+  end
+
+  def test_mention_at_end_of_parenthetical_sentence
+    @body = "(We're talking 'bout @ymendel.)"
+    assert_equal %w[ymendel], mentioned_usernames
+  end
+end

data/test/html/pipeline/plain_text_input_filter_test.rb

@@ -0,0 +1,22 @@
+require "test_helper"
+
+class HTML::Pipeline::PlainTextInputFilterTest < Test::Unit::TestCase
+  PlainTextInputFilter = HTML::Pipeline::PlainTextInputFilter
+
+  def test_fails_when_given_a_documentfragment
+    body = "<p>heyo</p>"
+    doc  = Nokogiri::HTML::DocumentFragment.parse(body)
+    assert_raise(TypeError) { PlainTextInputFilter.call(doc, {}) }
+  end
+
+  def test_wraps_input_in_a_div_element
+    doc = PlainTextInputFilter.call("howdy pahtner", {})
+    assert_equal "<div>howdy pahtner</div>", doc.to_s
+  end
+
+  def test_html_escapes_plain_text_input
+    doc = PlainTextInputFilter.call("See: <http://example.org>", {})
+    assert_equal "<div>See: &lt;http://example.org&gt;</div>",
+      doc.to_s
+  end
+end

data/test/html/pipeline/sanitization_filter_test.rb

@@ -0,0 +1,47 @@
+require "test_helper"
+
+class HTML::Pipeline::SanitizationFilterTest < Test::Unit::TestCase
+  SanitizationFilter = HTML::Pipeline::SanitizationFilter
+
+  def test_removing_script_tags
+    orig = %(<p><img src="http://github.com/img.png" /><script></script></p>)
+    html = SanitizationFilter.call(orig).to_s
+    assert_no_match /script/, html
+  end
+
+  def test_removing_style_tags
+    orig = %(<p><style>hey now</style></p>)
+    html = SanitizationFilter.call(orig).to_s
+    assert_no_match /style/, html
+  end
+
+  def test_removing_style_attributes
+    orig = %(<p style='font-size:1000%'>YO DAWG</p>)
+    html = SanitizationFilter.call(orig).to_s
+    assert_no_match /font-size/, html
+    assert_no_match /style/, html
+  end
+
+  def test_removing_script_event_handler_attributes
+    orig = %(<a onclick='javascript:alert(0)'>YO DAWG</a>)
+    html = SanitizationFilter.call(orig).to_s
+    assert_no_match /javscript/, html
+    assert_no_match /onclick/, html
+  end
+
+  def test_sanitizes_li_elements_not_contained_in_ul_or_ol
+    stuff = "a\n<li>b</li>\nc"
+    html  = SanitizationFilter.call(stuff).to_s
+    assert_equal "a\nb\nc", html
+  end
+
+  def test_does_not_sanitize_li_elements_contained_in_ul_or_ol
+    stuff = "a\n<ul><li>b</li></ul>\nc"
+    assert_equal stuff, SanitizationFilter.call(stuff).to_s
+  end
+
+  def test_github_specific_protocols_are_not_removed
+    stuff = '<a href="github-windows://spillthelog">Spill this yo</a> and so on'
+    assert_equal stuff, SanitizationFilter.call(stuff).to_s
+  end
+end

data/test/html/pipeline/toc_filter_test.rb

@@ -0,0 +1,47 @@
+require "test_helper"
+
+class HTML::Pipeline::TableOfContentsFilterTest < Test::Unit::TestCase
+  TocFilter = HTML::Pipeline::TableOfContentsFilter
+
+  def test_anchors_are_added_properly
+    orig = %(<h1>Ice cube</h1><p>Will swarm on any motherfucker in a blue uniform</p>)
+    assert_includes '<a name=', TocFilter.call(orig).to_s
+  end
+
+  def test_anchors_have_sane_names
+    orig = %(<h1>Dr Dre</h1><h1>Ice Cube</h1><h1>Eazy-E</h1><h1>MC Ren</h1>)
+    result = TocFilter.call(orig).to_s
+
+    assert_includes '"dr-dre"', result
+    assert_includes '"ice-cube"', result
+    assert_includes '"eazy-e"', result
+    assert_includes '"mc-ren"', result
+  end
+
+  def test_dupe_headers_have_unique_trailing_identifiers
+    orig = %(<h1>Straight Outta Compton</h1>
+             <h2>Dopeman</h2>
+             <h3>Express Yourself</h3>
+             <h1>Dopeman</h1>)
+
+    result = TocFilter.call(orig).to_s
+
+    assert_includes '"dopeman"', result
+    assert_includes '"dopeman-1"', result
+  end
+
+  def test_all_header_tags_are_found_when_adding_anchors
+    orig = %(<h1>"Funky President" by James Brown</h1>
+             <h2>"It's My Thing" by Marva Whitney</h2>
+             <h3>"Boogie Back" by Roy Ayers</h3>
+             <h4>"Feel Good" by Fancy</h4>
+             <h5>"Funky Drummer" by James Brown</h5>
+             <h6>"Ruthless Villain" by Eazy-E</h6>
+             <h7>"Be Thankful for What You Got" by William DeVaughn</h7>)
+
+    doc = TocFilter.call(orig)
+    assert_equal 6, doc.search('a').size
+  end
+end
+
+

data/test/test_helper.rb

@@ -0,0 +1,38 @@
+require 'bundler/setup'
+require 'html/pipeline'
+require 'test/unit'
+
+require 'active_support/core_ext/object/try'
+
+module TestHelpers
+  # Asserts that `needle` is not a member of `haystack`, where
+  # `haystack` is any object that responds to `include?`.
+  def assert_doesnt_include(needle, haystack, message = nil)
+    error = '<?> included in <?>'
+    message = build_message(message, error, needle.to_s, Array(haystack).map(&:to_s))
+
+    assert_block message do
+      !haystack.include?(needle)
+    end
+  end
+
+  # Asserts that `needle` is a member of `haystack`, where
+  # `haystack` is any object that responds to `include?`.
+  def assert_includes(needle, haystack, message = nil)
+    error = '<?> not included in <?>'
+    message = build_message(message, error, needle.to_s, Array(haystack).map(&:to_s))
+
+    assert_block message do
+      haystack.include?(needle)
+    end
+  end
+
+  # Asserts that two html fragments are equivalent. Attribute order
+  # will be ignored.
+  def assert_equal_html(expected, actual)
+    assert_equal Nokogiri::HTML::DocumentFragment.parse(expected).to_hash,
+                 Nokogiri::HTML::DocumentFragment.parse(actual).to_hash
+  end
+end
+
+Test::Unit::TestCase.send(:include, TestHelpers)

metadata

@@ -0,0 +1,214 @@
+--- !ruby/object:Gem::Specification
+name: html-pipeline-no-charlock
+version: !ruby/object:Gem::Version
+  version: 0.0.6
+  prerelease: 
+platform: ruby
+authors:
+- Ryan Tomayko
+- Jerry Cheung
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: gemoji
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: github-markdown
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: sanitize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rinku
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: escape_utils
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2'
+description: GitHub HTML processing filters and utilities. This version remove dependencies
+  on charlock_holmes to make it easy be installed on Windows
+email:
+- ryan@github.com
+- jerry@github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- html-pipeline-no-charlock.gemspec
+- html-pipeline.gemspec
+- lib/html/pipeline.rb
+- lib/html/pipeline/@mention_filter.rb
+- lib/html/pipeline/autolink_filter.rb
+- lib/html/pipeline/body_content.rb
+- lib/html/pipeline/camo_filter.rb
+- lib/html/pipeline/email_reply_filter.rb
+- lib/html/pipeline/emoji_filter.rb
+- lib/html/pipeline/filter.rb
+- lib/html/pipeline/https_filter.rb
+- lib/html/pipeline/image_max_width_filter.rb
+- lib/html/pipeline/markdown_filter.rb
+- lib/html/pipeline/plain_text_input_filter.rb
+- lib/html/pipeline/sanitization_filter.rb
+- lib/html/pipeline/syntax_highlight_filter.rb
+- lib/html/pipeline/text_filter.rb
+- lib/html/pipeline/textile_filter.rb
+- lib/html/pipeline/toc_filter.rb
+- lib/html/pipeline/version.rb
+- test/html/pipeline/autolink_filter_test.rb
+- test/html/pipeline/camo_filter_test.rb
+- test/html/pipeline/emoji_filter_test.rb
+- test/html/pipeline/image_max_width_filter_test.rb
+- test/html/pipeline/markdown_filter_test.rb
+- test/html/pipeline/mention_filter_test.rb
+- test/html/pipeline/plain_text_input_filter_test.rb
+- test/html/pipeline/sanitization_filter_test.rb
+- test/html/pipeline/toc_filter_test.rb
+- test/test_helper.rb
+homepage: https://github.com/doitian/html-pipeline
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -4163695188882842719
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -4163695188882842719
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Helpers for processing content through a chain of filters
+test_files:
+- test/html/pipeline/autolink_filter_test.rb
+- test/html/pipeline/camo_filter_test.rb
+- test/html/pipeline/emoji_filter_test.rb
+- test/html/pipeline/image_max_width_filter_test.rb
+- test/html/pipeline/markdown_filter_test.rb
+- test/html/pipeline/mention_filter_test.rb
+- test/html/pipeline/plain_text_input_filter_test.rb
+- test/html/pipeline/sanitization_filter_test.rb
+- test/html/pipeline/toc_filter_test.rb
+- test/test_helper.rb