upmark 0.2.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05a88fea05d73c8e0e6b6283687fde275582f030
-  data.tar.gz: ca3431e72f14140a5e558dc74d275390c010a423
+  metadata.gz: ebfa7a03ebe86bf8d70709fe99002518ee9f7237
+  data.tar.gz: be21325fe8ab8cd94d9ba4a4df1fcb0025bfb64b
 SHA512:
-  metadata.gz: 190e387ab1335b918dc81d002b14acd97ffe1c42b8181066018908cefc4fe9729fc29081e64908d2f5f94ead4836d22abb316078c3b74c8929ce0f1dab6dcaa4
-  data.tar.gz: a200693de5b909b23d4e385fa769dfd0588531621587e071b2bde4c4fe126033e66293cd02b93b8c5e05a61df6294b779f2403d83c45212c8a37a002a40648ee
+  metadata.gz: d4eab42d3640a2f9011daf71244c97036f9e92bce80cf351fe9eeda00b1535f0ef5a0471440ec1c4dd9d5c0ac8c76d9c0263661f9c2c6f9d1425a3845296c901
+  data.tar.gz: 3ed9e855887dbab065c586517669657e7c4cb56311b1e5237161111e64853043c2e44097f2bff95b5841ecd5fe3424cd29811940fc07fe8e860472a4ad77b92c

data/README.md

@@ -31,7 +31,7 @@ Upmark will convert the following (arbitrarily nested) HTML elements to Markdown
 * `em`
 * `p`
 * `a`
-* `h1`, `h2`, `h3`
+* `h1`, `h2`, `h3`, `h4`, `h5`, `h6`
 * `ul`
 * `ol`
 * `br`

data/lib/upmark.rb

@@ -24,11 +24,20 @@ module Upmark
     # The result is either a String or an Array.
     ast = ast.join if ast.is_a?(Array)
 
+    # Remove trailing whitespace
+    ast.gsub!(/ +$/,'')
+
+    # Compress bullet point lists
+    ast.gsub!(/^•\s*([^•\n]*)\n+(?=•)/,"* #{'\1'}\n")
+
     # Any more than two consecutive newline characters is superflous.
-    ast = ast.gsub(/\n(\s*\n)+/, "\n\n")
+    ast.gsub!(/\n(\s*\n)+/, "\n\n")
+
+    # Remove other bullet points
+    ast.gsub!(/^•\s*/,"* ")
 
     ast.strip
-  rescue Parslet::ParseFailed
-    raise Upmark::ParseFailed
+  rescue Parslet::ParseFailed => e
+    raise Upmark::ParseFailed.new('Parse failed', e)
   end
 end

data/lib/upmark/errors.rb

@@ -1,3 +1,14 @@
 module Upmark
-  ParseFailed = Class.new(StandardError)
+  class ParseFailed < StandardError
+
+    def initialize(message, cause)
+      @cause = cause
+      super(message)
+    end
+
+    def cause
+      @cause
+    end
+
+  end
 end

data/lib/upmark/parser/xml.rb

@@ -10,68 +10,81 @@ module Upmark
     class XML < Parslet::Parser
       root(:node)
 
-      rule(:node) {
+      rule(:node) do
         (
+          empty_element.as(:empty) |
           element.as(:element) |
           text.as(:text)
         ).repeat(0)
-      }
+      end
 
-      rule(:element) {
+      rule(:empty_element) do
+        start_tag.as(:start_tag) >>
+        match(/\s+/) >>
+        end_tag.as(:end_tag)
+      end
+
+      rule(:element) do
+        empty_br.as(:empty_tag) |
         (
           start_tag.as(:start_tag) >>
           node.as(:children) >>
           end_tag.as(:end_tag)
         ) |
         empty_tag.as(:empty_tag)
-      }
+      end
 
-      rule(:text) {
+      rule(:text) do
+        match(/\A[\s\n\t ]+\Z/m).absent? >> # ignore entirely empty strings
         match(/[^<>]/).repeat(1)
-      }
+      end
 
-      rule(:start_tag) {
+      rule(:start_tag) do
         str('<') >>
         name.as(:name) >>
         (space >> attribute).repeat.as(:attributes) >>
         space? >>
         str('>')
-      }
+      end
 
-      rule(:end_tag) {
+      rule(:end_tag) do
         str('</') >>
         name.as(:name) >>
         space? >>
         str('>')
-      }
+      end
+
+      rule(:empty_br) do
+        str('<') >> space? >> str('br').as(:name) >> space? >> str('>')
+      end
 
-      rule(:empty_tag) {
+      rule(:empty_tag) do
         str('<') >>
         name.as(:name) >>
         (space >> attribute).repeat.as(:attributes) >>
         space? >>
         str('/>')
-      }
+      end
 
-      rule(:name) {
+      rule(:name) do
         match(/[a-zA-Z_:]/) >> match(/[\w:\.-]/).repeat
-      }
+      end
 
-      rule(:attribute) {
+      rule(:attribute) do
         name.as(:name) >>
         str('=') >> (
           (str('"') >> double_quoted_attribute_value.as(:value) >> str('"')) | # double quotes
           (str("'") >> single_quoted_attribute_value.as(:value) >> str("'"))   # single quotes
         )
-      }
+      end
 
-      rule(:double_quoted_attribute_value) {
+      rule(:double_quoted_attribute_value) do
         (str('"').absent? >> (match(/[^<&]/) | entity_ref)).repeat
-      }
+      end
 
-      rule(:single_quoted_attribute_value) {
+      rule(:single_quoted_attribute_value) do
         (str("'").absent? >> (match(/[^<&]/) | entity_ref)).repeat
-      }
+      end
 
       rule(:entity_ref) { match("&") >> name >> match(";") }
 

data/lib/upmark/transform/markdown.rb

@@ -7,19 +7,49 @@ module Upmark
 
       rule(text: simple(:value)) { value.to_s }
 
+      # Pass all unmatched elements through.
+      rule(
+        element: {
+          name:       simple(:name),
+          attributes: subtree(:attributes),
+          children:   sequence(:children),
+          ignore:     simple(:ignore)
+        }
+      ) do |element|
+        attributes = map_attributes_subtree(element[:attributes])
+        children   = element[:children].join
+        name       = element[:name]
+
+        attributes_list =
+          if attributes.any?
+            " " + attributes.map {|name, value| %Q{#{name}="#{value}"} }.join(" ")
+          else
+            ""
+          end
+
+        if children.empty?
+          %Q{<#{name}#{attributes_list} />}
+        else
+          %Q{<#{name}#{attributes_list}>#{children}</#{name}>}
+        end
+      end
+
       def self.text(element)
-        element[:children].join.gsub(/(\n)+/, '\1')
+        element[:children].join.gsub(/(\n)[\n ]+/, '\1')
       end
 
       element(:p)  {|element| "#{text(element)}\n\n" }
       element(:h1) {|element| "# #{text(element)}" }
       element(:h2) {|element| "## #{text(element)}" }
       element(:h3) {|element| "### #{text(element)}" }
+      element(:h4) {|element| "#### #{text(element)}" }
+      element(:h5) {|element| "##### #{text(element)}" }
+      element(:h6) {|element| "###### #{text(element)}" }
       element(:li) {|element| "#{text(element)}" }
 
       element(:ul) do |element|
         children = element[:children].map {|value| value.strip != "" ? value : nil }.compact
-        children.map {|value| "* #{value}\n" }
+        children.map {|value| "* #{value.gsub(/^\s*•\s*/,'')}\n" }
       end
 
       element(:ol) do |element|
@@ -32,7 +62,11 @@ module Upmark
         href       = attributes[:href]
         title      = attributes[:title]
 
-        %Q{[#{text(element)}](#{href} "#{title}")}
+        if /^(?:http|mailto)/ =~ href
+          %Q{[#{text(element)}](#{href} "#{title}")}
+        else
+          text(element)
+        end
       end
 
       element(:img) do |element|
@@ -41,40 +75,19 @@ module Upmark
         title      = attributes[:title]
         alt_text   = attributes[:alt]
 
-        %Q{![#{alt_text}](#{href} "#{title}")}
+        if /^http/ =~ href
+          %Q{![#{alt_text}](#{href} "#{title}")}
+        else
+          "#{alt_text || title}"
+        end
       end
 
       element(:b, :strong) {|element| "**#{text(element)}**" }
       element(:i, :em)     {|element| "*#{text(element)}*" }
 
       element(:br) { "\n" }
+      rule(element: { name: "br"}) { "\n" }
 
-      # Pass all unmatched elements through.
-      rule(
-        element: {
-          name:       simple(:name),
-          attributes: subtree(:attributes),
-          children:   sequence(:children),
-          ignore:     simple(:ignore)
-        }
-      ) do |element|
-        attributes = map_attributes_subtree(element[:attributes])
-        children   = element[:children].join
-        name       = element[:name]
-
-        attributes_list =
-          if attributes.any?
-            " " + attributes.map {|name, value| %Q{#{name}="#{value}"} }.join(" ")
-          else
-            ""
-          end
-
-        if children.empty?
-          %Q{<#{name}#{attributes_list} />}
-        else
-          %Q{<#{name}#{attributes_list}>#{children}</#{name}>}
-        end
-      end
     end
   end
 end

data/lib/upmark/transform/normalise.rb

@@ -3,13 +3,26 @@ module Upmark
     # A transform class withich normalises start/end/empty tags into the
     # same structure.
     class Normalise < Parslet::Transform
+
+      rule(element: subtree(:invalid)) do
+        raise Upmark::ParseFailed.new('Invalid parse result', nil)
+      end
+
+      # Strip empty tags
+      rule(empty: subtree(:invalid)) do
+        ' '
+      end
+
       rule(
         element: {
           start_tag: {name: simple(:name), attributes: subtree(:attributes)},
-          end_tag:   {name: simple(:name)},
+          end_tag:   {name: simple(:end_tag_name)},
           children:  subtree(:children)
         }
       ) do
+        unless name == end_tag_name
+          raise Upmark::ParseFailed.new('Mismatched tags', nil)
+        end
         {
           element: {
             name:       name,
@@ -20,6 +33,21 @@ module Upmark
         }
       end
 
+      rule(
+        element: {
+          empty_tag: { name: simple(:name) }
+        }
+      ) do
+        {
+          element: {
+            name:       name,
+            attributes: [],
+            children:   [],
+            ignore:     false
+          }
+        }
+      end
+
       rule(
         element: {
           empty_tag: {name: simple(:name), attributes: subtree(:attributes)}
@@ -34,6 +62,7 @@ module Upmark
           }
         }
       end
+
     end
   end
 end

data/lib/upmark/transform/preprocess.rb

@@ -7,7 +7,7 @@ module Upmark
     class Preprocess < Parslet::Transform
       include TransformHelpers
 
-      element(:div, :table, :pre) do |element|
+      element(:div, :pre) do |element|
         {
           element: {
             name:       element[:name],
@@ -17,6 +17,37 @@ module Upmark
           }
         }
       end
+
+      element(:span) do |element|
+        element[:children]
+      end
+
+      # table content elements are stripped ignoring their spacing
+      element(:table, :thead, :tbody, :tfoot) do |element|
+        element[:children].reject! do |c|
+          Hash === c && c[:text].to_s =~ /\A[\n ]*\Z/m
+        end
+        element[:children]
+      end
+
+      # table content elements are stripped
+      element(:td, :th) do |element|
+        element[:children]
+      end
+
+      # table rows are treated as 'paragraph' blocks
+      element(:tr) do |element|
+        element[:children]
+          .select { |c| Array === c }
+          .map do |children|
+            children.map do |child|
+              if child[:text]
+                child[:text].to_s.gsub!(/^\n */,'')
+              end
+              child
+            end + ["\n"]
+          end + ["\n"]
+      end
     end
   end
 end

data/spec/acceptance/upmark_spec.rb

@@ -1,57 +1,102 @@
-require "spec_helper"
+RSpec.describe Upmark, ".convert" do
+  RSpec::Matchers.define :convert_to do |expected|
+    match do
+      actual == expected
+    end
 
-describe Upmark, ".convert" do
-  subject { Upmark.convert(html) }
+    def actual
+      @converted_actual ||= Upmark.convert(@actual)
+    end
 
-  context "<a>" do
-    let(:html) { <<-HTML.strip }
-<p><a href="http://helvetica.com/" title="art party organic">messenger <strong>bag</strong> skateboard</a></p>
-    HTML
+    diffable
+  end
 
-    it { should == <<-MD.strip }
-[messenger **bag** skateboard](http://helvetica.com/ "art party organic")
-    MD
+  context "<a>" do
+    specify 'converts to []()' do
+      expect(<<-HTML.strip
+        <p><a href="http://helvetica.com/" title="art party organic">messenger <strong>bag</strong> skateboard</a></p>
+      HTML
+      ).to convert_to <<-MD.strip
+        [messenger **bag** skateboard](http://helvetica.com/ "art party organic")
+      MD
+    end
   end
 
   context "<a> hard" do
-    let(:html) { <<-HTML.strip }
-<p><a href="http://jobs.latrobe.edu.au/jobDetails.asp?sJobIDs=545808&amp;sKeywords=business">Manager, Business Solutions</a></p>
-    HTML
-
-    it { should == <<-MD.strip }
+    specify 'converts as []()' do
+      expect(<<-HTML.strip
+        <p><a href="http://jobs.latrobe.edu.au/jobDetails.asp?sJobIDs=545808&amp;sKeywords=business">Manager, Business Solutions</a></p>
+      HTML
+      ).to convert_to <<-MD.strip
 [Manager, Business Solutions](http://jobs.latrobe.edu.au/jobDetails.asp?sJobIDs=545808&amp;sKeywords=business "")
-    MD
+      MD
+    end
   end
 
   context "<img>" do
-    let(:html) { <<-HTML.strip }
-<img src="http://helvetica.com/image.gif" title="art party organic" alt="messenger bag skateboard" />
-
-    HTML
-
-    it { should == <<-MD.strip }
+    specify 'converts as ![]()' do
+      expect(<<-HTML.strip
+        <img src="http://helvetica.com/image.gif" title="art party organic" alt="messenger bag skateboard" />
+      HTML
+      ).to convert_to <<-MD.strip
 ![messenger bag skateboard](http://helvetica.com/image.gif "art party organic")
-    MD
+      MD
+    end
   end
 
   context "<p>" do
-    let(:html) { <<-HTML.strip }
+    specify 'converts as plaintext' do
+      expect(<<-HTML.strip
+<p>• Bullet 1</p>
+<p>• Bullet 2</p>
 <p>messenger <strong>bag</strong> skateboard</p>
 
 <p>art party<br />
 organic</p>
-    HTML
 
-    it { should == <<-MD.strip }
+<p>art party<br>
+organic</p>
+
+<p> </p>
+<p><strong> </strong></p>
+
+<p>• Bullet 3</p>
+<p>• Bullet 4</p>
+<p>• Bullet 5</p>
+<p>• Bullet 6</p>
+<p>• Bullet 7</p>
+<p>Something else</p>
+      HTML
+      ).to convert_to <<-MD.strip
+* Bullet 1
+* Bullet 2
+
 messenger **bag** skateboard
 
 art party
 organic
-    MD
+
+art party
+organic
+
+* Bullet 3
+* Bullet 4
+* Bullet 5
+* Bullet 6
+* Bullet 7
+
+Something else
+      MD
+    end
+
+    it 'converts paragraph utf-8 bullet points to a markdown list' do
+      expect("<p>• Bullet 1</p><p>• Bullet 2</p>").to convert_to "* Bullet 1\n* Bullet 2"
+    end
   end
 
   context "<ul>" do
-    let(:html) { <<-HTML.strip }
+    specify 'converts as list' do
+      expect(<<-HTML.strip
 <ul>
   <li>messenger</li>
   <li><strong>bag</strong></li>
@@ -63,9 +108,13 @@ organic
   <li><p><strong>bag</strong></p></li>
   <li><p>skateboard</p></li>
 </ul>
-    HTML
 
-    it { should == <<-MD.strip }
+<ul>
+  <li>• Bullet 1</li>
+  <li>• Bullet 2</li>
+</ul>
+      HTML
+      ).to convert_to <<-MD.strip
 * messenger
 * **bag**
 * skateboard
@@ -75,11 +124,16 @@ organic
 * **bag**
 
 * skateboard
-    MD
+
+* Bullet 1
+* Bullet 2
+      MD
+    end
   end
 
   context "<ol>" do
-    let(:html) { <<-HTML.strip }
+    specify 'converts as numbered list' do
+      expect(<<-HTML.strip
 <ol>
   <li>messenger</li>
   <li><strong>bag</strong></li>
@@ -91,9 +145,8 @@ organic
   <li><p><strong>bag</strong></p></li>
   <li><p>skateboard</p></li>
 </ol>
-    HTML
-
-    it { should == <<-MD.strip }
+      HTML
+      ).to convert_to <<-MD.strip
 1. messenger
 2. **bag**
 3. skateboard
@@ -103,21 +156,29 @@ organic
 2. **bag**
 
 3. skateboard
-    MD
+      MD
+    end
   end
 
-  context "<h1>" do
-    let(:html) { <<-HTML.strip }
+  context "<h1>, <h2>, <h3>, <h4>, <h5>, <h6>" do
+    specify 'converts as #' do
+      expect(<<-HTML.strip
 <h1>messenger bag skateboard</h1>
 <h2>messenger bag skateboard</h2>
 <h3>messenger bag skateboard</h3>
-    HTML
-
-    it { should == <<-MD.strip }
+<h4>messenger bag skateboard</h4>
+<h5>messenger bag skateboard</h5>
+<h6>messenger bag skateboard</h6>
+      HTML
+      ).to convert_to <<-MD.strip
 # messenger bag skateboard
 ## messenger bag skateboard
 ### messenger bag skateboard
-    MD
+#### messenger bag skateboard
+##### messenger bag skateboard
+###### messenger bag skateboard
+      MD
+    end
   end
 
   context "block-level elements" do
@@ -127,25 +188,52 @@ organic
 <div id="tofu" class="art party">messenger <strong>bag</strong> skateboard</div>
       HTML
 
-      it { should == html }
+      specify 'are left alone' do
+        expect(html).to convert_to html
+      end
     end
 
     context "<table>" do
       let(:html) { <<-HTML.strip }
 <table>
   <tr>
-    <td>messenger</td>
+    <td><p><strong>messenger</strong></p></td>
+    <td><p>bag</p></td>
+  </tr>
+  <tr>
+    <td><p>messenger</p></td>
+    <td><p><strong>bag</strong></p></td>
   </tr>
   <tr>
-    <td><strong>bag</strong></td>
+    <td>skateboarding</td>
+    <td><p>is cool with all the kids<br/>
+      or something</p></td>
   </tr>
   <tr>
-    <td>skateboard</td>
+    <td><strong>Messenger bags</strong></td>
+    <td>are in with the hipsters though.</td>
   </tr>
 </table>
       HTML
 
-      it { should == html }
+      specify 'is converted to paragraphs' do
+        expect(html).to convert_to <<-MD.strip
+**messenger**
+
+bag
+
+messenger
+
+**bag**
+
+skateboarding
+is cool with all the kids
+or something
+
+**Messenger bags**
+are in with the hipsters though.
+        MD
+      end
     end
 
     context "<pre>" do
@@ -157,22 +245,31 @@ organic
 </pre>
       HTML
 
-      it { should == html }
+      specify 'are left alone' do
+        expect(html).to convert_to html
+      end
     end
   end
 
-  context "span-level elements" do
-    context "<span>" do
-      let(:html) { <<-HTML.strip }
+  context "<span> elements" do
+    specify 'are stripped' do
+      expect(<<-HTML.strip
 <span>messenger <strong>bag</strong> skateboard</span>
       HTML
-
-      it { should == <<-MD.strip }
-<span>messenger **bag** skateboard</span>
+      ).to convert_to <<-MD.strip
+messenger **bag** skateboard
       MD
     end
   end
 
+  context "plain text" do
+    it 'containing plain bullet points converts to markdown' do
+      expect(
+        "• Bullet 1\n• Bullet 2\n"
+      ).to convert_to "* Bullet 1\n* Bullet 2"
+    end
+  end
+
   context "unbalanced elements" do
     let(:html) { "<span><span>foo</span>" }
 
@@ -182,4 +279,14 @@ organic
       }.to raise_error(Upmark::ParseFailed)
     end
   end
+
+  context "unbalanced elements" do
+    let(:html) { "<p>foo</b>" }
+
+    it "should raise an exception" do
+      expect {
+        Upmark.convert(html)
+      }.to raise_error(Upmark::ParseFailed)
+    end
+  end
 end