devcenter-parser 1.0.0

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Heroku
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,27 @@
+# devcenter-parser
+
+Markdown parser used by [Heroku Dev Center](https://devcenter.heroku.com).
+
+Usage:
+
+```ruby
+    require 'devcenter-parser'
+
+    md = '[Dev Center](https://devcenter.heroku.com)'
+    flavour = :github # :github or :maruku
+    DevcenterParser.to_html(md, flavour)
+    # => "<p><a href=\"https://devcenter.heroku.com\">Dev Center</a></p>"
+
+    broken_md = '[foo](bar'
+    begin
+      DevcenterParser.to_html(broken_md, :maruku)
+    rescue DevcenterParser::InvalidMarkdownError => e
+      puts e.message # parser-dependent (sometimes cryptic) debugging info
+    end
+```
+
+## License
+See the LICENSE file included in the distribution.
+
+## Copyright
+Copyright (C) 2013 Heroku <raul@heroku.com>.

data/devcenter-parser.gemspec

@@ -0,0 +1,23 @@
+$:.unshift Dir.pwd
+require File.expand_path("./lib/devcenter-parser")
+
+Gem::Specification.new do |gem|
+  gem.name        = "devcenter-parser"
+  gem.version     = DevcenterParser::VERSION
+  gem.authors     = ["Raul Murciano"]
+  gem.email       = ["raul@heroku.com"]
+  gem.homepage    = "https://devcenter.heroku.com"
+  gem.summary     = "Parser for Heroku Dev Center's content"
+  gem.description = "Parser for Heroku Dev Center's content"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = %w{ lib }
+
+  gem.add_runtime_dependency 'maruku'
+  gem.add_runtime_dependency 'nokogiri'
+  gem.add_runtime_dependency 'redcarpet'
+  gem.add_runtime_dependency 'sanitize'
+
+  gem.add_development_dependency 'minitest', '>2.0'
+end

data/lib/devcenter-parser.rb

@@ -0,0 +1,132 @@
+require 'maruku'
+require 'redcarpet'
+require 'nokogiri'
+require 'uri'
+require 'sanitize'
+
+module DevcenterParser
+
+  VERSION = '1.0.0'
+
+  AVAILABLE_FLAVOURS = [:github, :maruku]
+
+  class InvalidMarkdownError < Exception; end
+  class InvalidRawHTMLError < Exception; end
+  class UnknownFlavourError < Exception; end
+
+  def self.to_html(markdown, flavour)
+    html = to_unsanitized_html(markdown, flavour.to_sym)
+    sanitize(html)
+  end
+
+  def self.to_unsanitized_html(markdown, flavour)
+    doc = case flavour.to_sym
+          when :maruku
+            html = Maruku.new(markdown, :on_error => :raise).to_html
+            doc = Nokogiri::HTML::DocumentFragment.parse(html)
+            maruku_code_blocks(doc)
+            maruku_underscores_to_dashes_in_subheader_anchors(doc)
+          when :github
+            html = github_parser.render(markdown.to_s)
+            doc = Nokogiri::HTML::DocumentFragment.parse(html)
+            github_parse_special_blocks(doc)
+            github_underscores_to_dashes_in_subheader_anchors(doc)
+          else
+            raise UnknownFlavourError, "Markdown flavour '#{flavour}' not supported"
+          end
+    html = doc.to_html(:encoding => 'utf-8')
+    verify_raw_html(html)
+    html
+  rescue InvalidRawHTMLError => e
+    raise InvalidMarkdownError, e.message
+  rescue => e
+    raise InvalidMarkdownError, parse_maruku_error(e.message)
+  end
+
+  def self.sanitize(html)
+    Sanitize.clean(html, sanitize_config)
+  end
+
+  private
+
+  def self.github_parser
+    @@github_parser ||= Redcarpet::Markdown.new(Redcarpet::Render::HTML, fenced_code_blocks: true)
+  end
+
+  def self.sanitize_config
+    return @@sanitize_config if defined?(@@sanitize_config)
+    config = Sanitize::Config::RELAXED
+    config[:attributes][:all] += %w{ id class style name width height border align }
+    config[:attributes]['a'] += %w{ target }
+    config[:elements] += %w{ div span hr tt }
+
+    # embedded videos
+    config[:attributes][:all] += %w{ value src type allowscriptaccess allowfullscreen }
+    config[:elements] += %w{ object param embed }
+    config[:add_attributes] = {
+      'object' => {'allowscriptaccess' => 'never'},
+      'embed' => {'allowscriptaccess' => 'never'},
+      'param' => {'allowscriptaccess' => 'never'}
+    }
+
+    @@sanitize_config = config.merge({remove_contents: true, allow_comments: true})
+  end
+
+  def self.maruku_code_blocks(doc)
+    doc.css('pre>code').each do |node|
+      if match = node.content.match(/\A\s*:::\s*(\w+)/)
+        lang = match[1]
+        node.content = node.content.gsub(/\A\s*:::\s*\w+\n/, '')
+        node['class'] = lang
+      end
+    end
+    doc
+  end
+
+  def self.maruku_underscores_to_dashes_in_subheader_anchors(doc)
+    doc.css("h2,h3,h4,h5,h6").each do |node|
+      if node.attributes['id'] && node.attributes['id'].value
+        node.attributes['id'].value = node.attributes['id'].value.gsub(/_+/,'-')
+      end
+    end
+    doc
+  end
+
+  def self.github_underscores_to_dashes_in_subheader_anchors(doc)
+    doc.css("h2,h3,h4,h5,h6").each do |node|
+      node['id'] = node.content.to_s.downcase.gsub(/\W/, '-')
+    end
+    doc
+  end
+
+  def self.github_parse_special_blocks(doc)
+    doc.css('blockquote>p:first').each do |node|
+      if match = node.inner_html.match(/\A\W*(callout|warning|note)\W/)
+        node.parent.name = 'div'
+        node.parent['class'] = match[1]
+        node.inner_html = node.inner_html.gsub(/\A\W*(callout|warning|note)\W/, '')
+      end
+    end
+  end
+
+  def self.verify_raw_html(html)
+    raise(InvalidRawHTMLError, parse_raw_html_error(html)) if invalid_raw_html?(html)
+  end
+
+  def self.invalid_raw_html?(html)
+    html.to_s.include?('markdown-html-error')
+  end
+
+  def self.parse_maruku_error(error_message)
+    lines = error_message.to_s.split("\n")
+    return lines unless lines.size > 1
+    msg = lines[4].gsub(/\A\|(\s)+|EOF\Z/,'').strip
+    code = lines[6].gsub(/\A\|(\s)+|EOF\Z/,'').strip
+    "#{msg} in \"#{code}\""
+  end
+
+  def self.parse_raw_html_error(html)
+    broken_html = html.match(/REXML could not parse this XML\/HTML\:(.+)<\/pre>/m)[1].strip rescue nil
+    broken_html.nil? ? "Contains broken raw HTML." : "This raw HTML is invalid: #{CGI.unescapeHTML(broken_html)}"
+  end
+end

data/test/devcenter-parser_test.rb

@@ -0,0 +1,207 @@
+require 'minitest/autorun'
+require_relative '../lib/devcenter-parser'
+
+
+describe 'DevcenterParser' do
+
+  describe '.to_unsanitized_html' do
+    it 'maintains script tags' do
+      md = '<script>alert("hi")</script>'
+      assert_parsing_unsanitized_result md, :maruku, md
+      assert_parsing_unsanitized_result md, :github, '<script>alert("hi")</script>'
+    end
+  end
+
+  describe '.to_html' do
+
+    it 'raises InvalidMarkdownError when parsing invalid markdown' do
+      md = '[foo](bar'
+      assert_raises DevcenterParser::InvalidMarkdownError do
+        DevcenterParser.to_html(md, :maruku)
+      end
+    end
+
+    it 'respects existing ids' do
+      md = '<strong id="foo">clean</strong>'
+      assert_maruku_result md, '<strong id="foo">clean</strong>'
+    end
+
+    it 'removes script tags and their content' do
+      md = '<strong>clean<script>alert("hack!")</script></strong>'
+      assert_maruku_result md, '<strong>clean</strong>'
+    end
+
+    it 'github markdown includes ids in subheaders' do
+      md = <<-MARKDOWN
+## Foo Bar Header 123
+
+Foo bar content
+    MARKDOWN
+      assert DevcenterParser.to_html(md, :github).include?('<h2 id="foo-bar-header-123">Foo Bar Header 123</h2>')
+    end
+
+    it 'maruku markdown includes ids in subheaders' do
+      md = <<-MARKDOWN
+## Foo Bar Header 123
+
+Foo bar content
+      MARKDOWN
+      assert DevcenterParser.to_html(md, :maruku).include?('<h2 id="foo-bar-header-123">Foo Bar Header 123</h2>')
+    end
+
+    it 'github markdown supports regular block quotes without callout|warning|note' do
+      md = <<-MARKDOWN
+Testing
+
+> not a callout
+> **strong**
+> normal
+
+And that's it.
+      MARKDOWN
+
+      html = <<-HTML
+<p>Testing</p>
+
+<blockquote>
+<p>not a callout
+<strong>strong</strong>
+normal</p>
+</blockquote>
+
+<p>And that's it.</p>
+      HTML
+
+      assert_github_result(md, html)
+
+      md = <<-MARKDOWN
+Testing
+
+> calloutnonono
+> **strong**
+> normal
+
+And that's it.
+      MARKDOWN
+
+      html = <<-HTML
+<p>Testing</p>
+
+<blockquote>
+<p>calloutnonono
+<strong>strong</strong>
+normal</p>
+</blockquote>
+
+<p>And that's it.</p>
+      HTML
+
+      assert_github_result(md, html)
+    end
+
+    it 'github markdown supports "> callout" and ">callout" and parses inner markdown' do
+      mds = []
+      mds << <<-MARKDOWN
+Testing
+
+> callout
+> **strong**
+> normal
+
+And that's it.
+      MARKDOWN
+
+      mds << <<-MARKDOWN
+Testing
+
+>callout
+>**strong**
+>normal
+
+And that's it.
+      MARKDOWN
+
+      html = <<-HTML
+<p>Testing</p>
+
+<div class="callout">
+<p><strong>strong</strong>
+normal</p>
+</div>
+
+<p>And that's it.</p>
+      HTML
+
+      mds.each do |md|
+        assert_github_result(md, html)
+      end
+    end
+
+    it 'github markdown supports "> callout" and ">callout", parses inner markdown and allows paragraphs' do
+      mds = []
+      mds << <<-MARKDOWN
+Testing
+
+> callout
+> **strong**
+
+> normal
+
+And that's it.
+      MARKDOWN
+
+      mds << <<-MARKDOWN
+Testing
+
+>callout
+>**strong**
+
+>normal
+
+And that's it.
+      MARKDOWN
+
+      html = <<-HTML
+<p>Testing</p>
+
+<div class="callout">
+<p><strong>strong</strong></p>
+
+<p>normal</p>
+</div>
+
+<p>And that's it.</p>
+      HTML
+
+      mds.each do |md|
+        assert_github_result(md, html)
+      end
+    end
+
+  end
+
+  # helpers
+
+  def assert_all_flavours_result(md, expected)
+    [:github, :maruku].each { |flavour| assert_parsing_result(md, flavour, expected) }
+  end
+
+  def assert_maruku_result(md, expected)
+    assert_parsing_result md, :maruku, expected
+  end
+
+  def assert_github_result(md, expected)
+    assert_parsing_result md, :github, expected
+  end
+
+  def assert_parsing_result(md, flavour, expected)
+    result = DevcenterParser.to_html(md, flavour)
+    assert_equal expected.strip, result.strip, "Failed when parsing\n#{md}\nwith the #{flavour} flavour.\n\nExpected:\n#{expected}\n\nActual result:\n#{result}\n\n"
+  end
+
+  def assert_parsing_unsanitized_result(md, flavour, expected)
+    result = DevcenterParser.to_unsanitized_html(md, flavour)
+    assert_equal expected.strip, result.strip, "Failed when parsing on unsanitized mode\n#{md}\nwith the #{flavour} flavour.\n\nExpected:\n#{expected}\n\nActual result:\n#{result}\n\n"
+  end
+
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: devcenter-parser
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Raul Murciano
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: maruku
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sanitize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Parser for Heroku Dev Center's content
+email:
+- raul@heroku.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- devcenter-parser.gemspec
+- lib/devcenter-parser.rb
+- test/devcenter-parser_test.rb
+homepage: https://devcenter.heroku.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Parser for Heroku Dev Center's content
+test_files:
+- test/devcenter-parser_test.rb