markitdown 0.2.1 → 0.3.0

data/README.md

@@ -1,4 +1,4 @@
-# Markitdown [![Build Status](https://secure.travis-ci.org/cpetersen/markitdown.png)](http://travis-ci.org/cpetersen/markitdown)
+# Markitdown [![Build Status](https://secure.travis-ci.org/cpetersen/markitdown.png)](http://travis-ci.org/cpetersen/markitdown) [![Coverage Status](https://coveralls.io/repos/cpetersen/markitdown/badge.png?branch=master)](https://coveralls.io/r/cpetersen/markitdown?branch=master)
 
 Markitdown is a Ruby library that converts HTML to Markdown. It's powered by Nokogiri. It supports:
 

data/lib/markitdown.rb

@@ -4,20 +4,20 @@ require "markitdown/version"
 require "nokogiri"
 
 module Markitdown
-  def self.from_html(html)
-    from_nokogiri(Nokogiri::XML(html).root)
+  def self.from_html(html, language_classifier=nil)
+    from_nokogiri(Nokogiri::XML(html).root, language_classifier)
   end
 
-  def self.from_nokogiri(node)
+  def self.from_nokogiri(node, language_classifier=nil)
     # gsub(/\n\s+\n/,"\n\n") - remove lines with nothing but space characters
     # gsub(/\n{2,}/,"\n\n") - collapse any series of more an than 2 new lines down to 2
     # gsub(/\t+/," ") - collapse consecutive tabs down to a single space. I use tabs to pad divs and span, this causes multiple nested spans and divs to ultimately be surrounded by a single space.
     # gsub(/ ([\.\?])/,'\1') - removes a space before a period or question mark. Things like links get surrounded by spaces. If they appear at the end of a sentence, this makes sure the punctation isn't off.
-    self.parse_node(node).flatten.compact.join.gsub(/\n\s+\n/,"\n\n").gsub(/\n{2,}/,"\n\n").gsub(/( > \n){2,}/,"\n > \n > ").gsub(/\t+/," ").gsub(/ ([\.\?])/,'\1').gsub(/\s*END_TAG\((.{1,3})\)/, "\\1")
+    self.parse_node(node, [], language_classifier).flatten.compact.join.gsub(/\n\s+\n/,"\n\n").gsub(/\n{2,}/,"\n\n").gsub(/( > \n){2,}/,"\n > \n > ").gsub(/\t+/," ").gsub(/ ([\.\?])/,'\1').gsub(/\s*END_TAG\((.{1,3})\)/, "\\1")
   end
 
   private
-  def self.parse_node(node, states=[])
+  def self.parse_node(node, states=[], language_classifier=nil)
     results=[]
     after = nil
     states.unshift node.name.downcase
@@ -141,7 +141,13 @@ module Markitdown
       results << node.text.strip.gsub("\n","").gsub(/ {2,}/," ")
     when "code"
       if node.text.include?("\n")
-        results << "\n\n```\n#{node.text.gsub(/^\n/,"").gsub(/\n\s*$/,"")}\n```\n\n"
+        text = node.text.gsub(/^\n/,"").gsub(/\n\s*$/,"")
+        if language_classifier
+          language = language_classifier.classify(text)
+          results << "\n\n```#{language}\n#{text}\n```\n\n"
+        else          
+          results << "\n\n```\n#{text}\n```\n\n"
+        end
       else
         results << " `#{node.text}` "
       end
@@ -171,7 +177,7 @@ module Markitdown
 
     if recurse
       node.children.each do |child|
-        contents = self.parse_node(child, states)
+        contents = self.parse_node(child, states, language_classifier)
         contents = contents.flatten.compact.join.strip if strip_content
         contents = [contents].flatten.compact.join.gsub("\n", " ") if flatten_content
         results << contents

data/lib/markitdown/version.rb

@@ -1,3 +1,3 @@
 module Markitdown
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/markitdown.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency('nokogiri')
   gem.add_development_dependency('rake')
   gem.add_development_dependency('rspec')
+  gem.add_development_dependency('coveralls')
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/spec/code.html

@@ -0,0 +1,38 @@
+<div>
+  This is an html block
+  <code>
+    &lt;table&gt;
+      &lt;thead&gt;
+        &lt;tr&gt;
+          &lt;th&gt;Column 1&lt;/th&gt;
+          &lt;th&gt;Column 2&lt;/th&gt;
+          &lt;th&gt;Column 3&lt;/th&gt;
+        &lt;/tr&gt;
+      &lt;/thead&gt;
+      &lt;tbody&gt;
+        &lt;tr&gt;
+          &lt;td&gt;Value 1&lt;/td&gt;
+          &lt;td&gt;Value 2&lt;/td&gt;
+          &lt;td&gt;Value 3&lt;/td&gt;
+        &lt;/tr&gt;
+        &lt;tr&gt;
+          &lt;td&gt;Value 1a&lt;/td&gt;
+          &lt;td&gt;Value 2a&lt;/td&gt;
+          &lt;td&gt;Value 3a&lt;/td&gt;
+        &lt;/tr&gt;
+      &lt;/tbody&gt;
+    &lt;/table&gt;
+  </code>
+
+  This is a ruby block
+  <code>
+    # GET /blogs/1
+    # GET /blogs/1.json
+    def show
+      respond_to do |format|
+        format.html # show.html.erb
+        format.json { render json: @blog }
+      end
+    end
+  </code>
+</div>

data/spec/code_spec.rb

@@ -0,0 +1,25 @@
+require 'markitdown'
+require 'spec_helper'
+
+describe Markitdown do  
+  context "When parsing codeblocks" do
+    let(:html) { File.read("spec/code.html") }
+
+    context "and not guessing the language" do
+      let(:markdown) { File.read("spec/code_without_language.markdown") }
+
+      it "should produce valid markdown" do
+        Markitdown.from_html(html).should == markdown
+      end
+    end
+
+    context "and guessing the language" do
+      let(:markdown) { File.read("spec/code_with_language.markdown") }
+      let(:classifier) { TestLanguageClassifier.new }
+
+      it "should produce valid markdown" do
+        Markitdown.from_html(html, classifier).should == markdown
+      end
+    end
+  end
+end

data/spec/code_with_language.markdown

@@ -0,0 +1,40 @@
+ This is an html block
+
+```html
+    <table>
+      <thead>
+        <tr>
+          <th>Column 1</th>
+          <th>Column 2</th>
+          <th>Column 3</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>Value 1</td>
+          <td>Value 2</td>
+          <td>Value 3</td>
+        </tr>
+        <tr>
+          <td>Value 1a</td>
+          <td>Value 2a</td>
+          <td>Value 3a</td>
+        </tr>
+      </tbody>
+    </table>
+```
+
+This is a ruby block
+
+```ruby
+    # GET /blogs/1
+    # GET /blogs/1.json
+    def show
+      respond_to do |format|
+        format.html # show.html.erb
+        format.json { render json: @blog }
+      end
+    end
+```
+
+ 

data/spec/code_without_language.markdown

@@ -0,0 +1,40 @@
+ This is an html block
+
+```
+    <table>
+      <thead>
+        <tr>
+          <th>Column 1</th>
+          <th>Column 2</th>
+          <th>Column 3</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>Value 1</td>
+          <td>Value 2</td>
+          <td>Value 3</td>
+        </tr>
+        <tr>
+          <td>Value 1a</td>
+          <td>Value 2a</td>
+          <td>Value 3a</td>
+        </tr>
+      </tbody>
+    </table>
+```
+
+This is a ruby block
+
+```
+    # GET /blogs/1
+    # GET /blogs/1.json
+    def show
+      respond_to do |format|
+        format.html # show.html.erb
+        format.json { render json: @blog }
+      end
+    end
+```
+
+ 

data/spec/doc_spec.rb

@@ -1,4 +1,5 @@
 require 'markitdown'
+require 'spec_helper'
 
 describe Markitdown do  
   context "When parsing a document" do

data/spec/nesting_spec.rb

@@ -1,4 +1,5 @@
 require 'markitdown'
+require 'spec_helper'
 
 describe Markitdown do
   context "when parsing nested ordered lists" do

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require 'coveralls'
+
+class TestLanguageClassifier
+  def classify(code)
+    if code
+      if code.match /<table>/
+        return "html"
+      elsif code.match /def/
+        return "ruby"
+      end
+    end
+  end
+end
+
+Coveralls.wear!

data/spec/table_spec.rb

@@ -1,4 +1,5 @@
 require 'markitdown'
+require 'spec_helper'
 
 describe Markitdown do  
   context "When parsing a table with a thead and tbody" do

data/spec/tag_spec.rb

@@ -1,4 +1,5 @@
 require 'markitdown'
+require 'spec_helper'
 
 describe Markitdown do
   context "When parsing a paragraph" do
@@ -166,7 +167,7 @@ describe Markitdown do
 
     it "should return valid markdown with spaces" do
       pending "Still need to figure out leading spaces for <sup> elements"
-      Markitdown.from_html(html).should == "This ^(is a) test"
+      # Markitdown.from_html(html).should == "This ^(is a) test"
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: markitdown
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-06 00:00:00.000000000 Z
+date: 2013-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -59,6 +59,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A library that uses Nokogiri to parse HTML and produce Markdown
 email:
 - christopher.petersen@gmail.com
@@ -77,12 +93,17 @@ files:
 - markitdown.gemspec
 - spec/asmartbear.html
 - spec/asmartbear.markdown
+- spec/code.html
+- spec/code_spec.rb
+- spec/code_with_language.markdown
+- spec/code_without_language.markdown
 - spec/doc.html
 - spec/doc.markdown
 - spec/doc_spec.rb
 - spec/evernote.markdown
 - spec/evernote.xml
 - spec/nesting_spec.rb
+- spec/spec_helper.rb
 - spec/table.html
 - spec/table.markdown
 - spec/table2.html
@@ -103,7 +124,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2638333383769331236
+      hash: -1182280358588269194
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -112,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2638333383769331236
+      hash: -1182280358588269194
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -122,12 +143,17 @@ summary: Converts HTML to Markdown
 test_files:
 - spec/asmartbear.html
 - spec/asmartbear.markdown
+- spec/code.html
+- spec/code_spec.rb
+- spec/code_with_language.markdown
+- spec/code_without_language.markdown
 - spec/doc.html
 - spec/doc.markdown
 - spec/doc_spec.rb
 - spec/evernote.markdown
 - spec/evernote.xml
 - spec/nesting_spec.rb
+- spec/spec_helper.rb
 - spec/table.html
 - spec/table.markdown
 - spec/table2.html