upmark 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rvmrc

data/.rspec

@@ -0,0 +1 @@
+--colour

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in upmark.gemspec
+gemspec

data/LICENSE.md

@@ -0,0 +1,20 @@
+Copyright (c) 2011 The Conversation Media Group
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,19 @@
+# Upmark
+
+A HTML to Markdown converter.
+
+## Installation
+
+    gem install upmark
+
+## Usage
+
+    require "upmark"
+
+    html = %q{<p>messenger <strong>bag</strong> skateboard</p>}
+    markdown = Upmark.convert(html)
+    puts markdown
+
+## License
+
+Upmark is Copyright (c) 2011 The Conversation Media Group and distributed under the MIT license.

data/Rakefile

@@ -0,0 +1,19 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+namespace :spec do
+  desc "Run acceptance specs"
+  RSpec::Core::RakeTask.new(:acceptance) do |t|
+    t.pattern = "./spec/acceptance/**/*_spec.rb"
+  end
+
+  desc "Run unit specs"
+  RSpec::Core::RakeTask.new(:unit) do |t|
+    t.pattern = "./spec/unit/**/*_spec.rb"
+  end
+
+  desc "Run unit and acceptance specs"
+  task :all => [:"spec:unit", :"spec:acceptance"]
+end
+
+task default: :"spec:all"

data/bin/upmark

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+$:.push File.expand_path("../../lib", __FILE__)
+
+require "upmark"
+
+if ARGV.length > 0
+  ARGV.each do |arg|
+    puts Upmark.convert(File.read(arg))
+  end
+else
+  puts Upmark.convert(ARGF.read)
+end
+

data/lib/core_ext/array.rb

@@ -0,0 +1,9 @@
+class Array
+  def map_with_index
+    result = []
+    each_with_index do |element, index|
+      result << yield(element, index)
+    end
+    result
+  end
+end

data/lib/upmark/parser/xml.rb

@@ -0,0 +1,76 @@
+module Upmark
+  module Parser
+    # The XML parser class.
+    #
+    # Parses a XML document into an abstract syntax tree (AST).
+    #
+    # It's worth referring to the XML spec:
+    #   http://www.w3.org/TR/2000/REC-xml-20001006
+    #
+    class XML < Parslet::Parser
+      root(:content)
+
+      rule(:content) {
+        (
+          element.as(:element) |
+          text.as(:text)
+        ).repeat(0)
+      }
+
+      rule(:element) {
+        (
+          start_tag.as(:start_tag) >>
+          content.as(:content) >>
+          end_tag.as(:end_tag)
+        ) |
+        empty_tag.as(:empty_tag)
+      }
+
+      rule(:text) {
+        match(/[^<>]/).repeat(1)
+      }
+
+      rule(:start_tag) {
+        str('<') >>
+        name.as(:name) >>
+        (space >> attribute).repeat.as(:attributes) >>
+        space? >>
+        str('>')
+      }
+
+      rule(:end_tag) {
+        str('</') >>
+        name.as(:name) >>
+        space? >>
+        str('>')
+      }
+
+      rule(:empty_tag) {
+        str('<') >>
+        name.as(:name) >>
+        (space >> attribute).repeat.as(:attributes) >>
+        space? >>
+        str('/>')
+      }
+
+      rule(:name) {
+        match(/[a-zA-Z_:]/) >> match(/[\w:\.-]/).repeat
+      }
+
+      rule(:attribute) {
+        name.as(:name) >>
+        str('=') >> (
+          (str('"') >> attribute_value.as(:value) >> str('"')) | # double quotes
+          (str("'") >> attribute_value.as(:value) >> str("'"))   # single quotes
+        )
+      }
+
+      rule(:attribute_value) {
+        (match(/['"]/).absent? >> match(/[^<&]/)).repeat
+      }
+
+      rule(:space)  { match(/\s/).repeat(1) }
+      rule(:space?) { space.maybe }
+    end
+  end
+end

data/lib/upmark/transform/markdown.rb

@@ -0,0 +1,84 @@
+module Upmark
+  module Transform
+    # The Markdown transform class.
+    #
+    # Transforms an abstract syntax tree (AST) into a Markdown document.
+    #
+    class Markdown < Parslet::Transform
+      def self.tag(tag_name)
+        tag_name = tag_name.to_s.downcase
+        {
+          tag:     {name: tag_name, attributes: subtree(:attributes)},
+          content: sequence(:values)
+        }
+      end
+
+      def self.map_attributes_subtree(ast)
+        ast.inject({}) do |hash, attribute|
+          hash[attribute[:name].to_sym] = attribute[:value]
+          hash
+        end
+      end
+
+      rule(element: subtree(:values)) { values }
+
+      rule(text: simple(:value)) { value.to_s }
+
+      rule(tag(:p))      { "#{values.join}\n\n" }
+      rule(tag(:strong)) { "**#{values.join}**" }
+      rule(tag(:em))     { "*#{values.join}*" }
+      rule(tag(:li))     { "#{values.join}" }
+      rule(tag(:h1))     { "# #{values.join}" }
+      rule(tag(:h2))     { "## #{values.join}" }
+      rule(tag(:h3))     { "### #{values.join}" }
+      rule(tag(:br))     { "\n" }
+
+      rule(tag(:ul)) do |dictionary|
+        values = dictionary[:values].map {|value| value.strip != "" ? value : nil }.compact
+        values.map {|value| "* #{value}\n" }
+      end
+
+      rule(tag(:ol)) do |dictionary|
+        values = dictionary[:values].map {|value| value.strip != "" ? value : nil }.compact
+        values.map_with_index {|value, i| "#{i + 1}. #{value}\n" }
+      end
+
+      rule(tag(:a)) do |dictionary|
+        attributes = map_attributes_subtree(dictionary[:attributes])
+        href       = attributes[:href]
+        title      = attributes[:title]
+        values     = dictionary[:values].join
+
+        %Q{[#{values}](#{href} "#{title}")}
+      end
+
+      rule(tag(:img)) do |dictionary|
+        attributes = map_attributes_subtree(dictionary[:attributes])
+        href       = attributes[:src]
+        title      = attributes[:title]
+        alt_text   = attributes[:alt]
+
+        %Q{![#{alt_text}](#{href} "#{title}")}
+      end
+
+      # Catch-all rule to pass all tags through.
+      rule(
+        tag:     {name: simple(:tag_name), attributes: subtree(:attributes)},
+        content: sequence(:values)
+      ) do |dictionary|
+        attributes = map_attributes_subtree(dictionary[:attributes])
+        values     = dictionary[:values].join
+        tag_name   = dictionary[:tag_name]
+
+        attributes_list =
+          if attributes.any?
+            " " + attributes.map {|name, value| %Q{#{name}="#{value}"} }.join(" ")
+          else
+            ""
+          end
+
+        %Q{<#{tag_name}#{attributes_list}>#{values}</#{tag_name}>}
+      end
+    end
+  end
+end

data/lib/upmark/transform/preprocess.rb

@@ -0,0 +1,25 @@
+module Upmark
+  module Transform
+    class Preprocess < Parslet::Transform
+      rule(
+        start_tag: {name: simple(:tag_name), attributes: subtree(:attributes)},
+        end_tag:   {name: simple(:tag_name)},
+        content:   subtree(:values)
+      ) do
+        {
+          tag:     {name: tag_name, attributes: attributes},
+          content: values
+        }
+      end
+
+      rule(
+        empty_tag: {name: simple(:tag_name), attributes: subtree(:attributes)}
+      ) do
+        {
+          tag:     {name: tag_name, attributes: attributes},
+          content: []
+        }
+      end
+    end
+  end
+end

data/lib/upmark/version.rb

@@ -0,0 +1,3 @@
+module Upmark
+  VERSION = "0.0.1"
+end

data/lib/upmark.rb

@@ -0,0 +1,28 @@
+require "parslet"
+
+require "core_ext/array"
+
+require "upmark/parser/xml"
+require "upmark/transform/markdown"
+require "upmark/transform/preprocess"
+require "upmark/version"
+
+module Upmark
+  def self.convert(html)
+    xml        = Parser::XML.new
+    preprocess = Transform::Preprocess.new
+    markdown   = Transform::Markdown.new
+
+    ast = xml.parse(html.strip)
+    ast = preprocess.apply(ast)
+    ast = markdown.apply(ast)
+
+    # The result is either a String or an Array.
+    ast = ast.join if ast.is_a?(Array)
+
+    # Any more than two consecutive newline characters is superflous.
+    ast = ast.gsub(/\n(\s*\n)+/, "\n\n")
+
+    ast.strip
+  end
+end

data/spec/acceptance/upmark_spec.rb

@@ -0,0 +1,79 @@
+require "spec_helper"
+
+describe Upmark, ".convert" do
+  subject { Upmark.convert(html) }
+
+  let(:html) { <<-HTML.strip }
+<h1>messenger bag skateboard</h1>
+
+<p>messenger bag skateboard</p>
+
+<p>messenger <em>bag</em> <strong>skateboard</strong></p>
+
+<p><a href="http://helvetica.com/" title="art party organic">messenger <strong>bag</strong> skateboard</a></p>
+
+<img src="http://helvetica.com/image.gif" title="art party organic" alt="messenger bag skateboard" />
+
+<ul>
+  <li>messenger</li>
+  <li><strong>bag</strong></li>
+  <li>skateboard</li>
+</ul>
+
+<ul>
+  <li><p>messenger</p></li>
+  <li><p><strong>bag</strong></p></li>
+  <li><p>skateboard</p></li>
+</ul>
+
+<ol>
+  <li>messenger</li>
+  <li><strong>bag</strong></li>
+  <li>skateboard</li>
+</ol>
+
+<ol>
+  <li><p>messenger</p></li>
+  <li><p><strong>bag</strong></p></li>
+  <li><p>skateboard</p></li>
+</ol>
+
+<div>messenger <strong>bag</strong> skateboard</div>
+<div id="tofu" class="art party">messenger <strong>bag</strong> skateboard</div>
+  HTML
+
+  it { should == <<-MD.strip }
+# messenger bag skateboard
+
+messenger bag skateboard
+
+messenger *bag* **skateboard**
+
+[messenger **bag** skateboard](http://helvetica.com/ "art party organic")
+
+![messenger bag skateboard](http://helvetica.com/image.gif "art party organic")
+
+* messenger
+* **bag**
+* skateboard
+
+* messenger
+
+* **bag**
+
+* skateboard
+
+1. messenger
+2. **bag**
+3. skateboard
+
+1. messenger
+
+2. **bag**
+
+3. skateboard
+
+<div>messenger **bag** skateboard</div>
+<div id="tofu" class="art party">messenger **bag** skateboard</div>
+  MD
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require "parslet/rig/rspec"
+require "rspec"
+require "upmark"

data/spec/unit/lib/upmark/parser/xml_spec.rb

@@ -0,0 +1,219 @@
+require "spec_helper"
+
+describe Upmark::Parser::XML do
+  let(:parser) { Upmark::Parser::XML.new }
+
+  context "#content" do
+    subject { parser.content }
+
+    it { should parse "" }
+    it { should parse "messenger bag skateboard" }
+    it { should parse "<p>messenger bag skateboard</p>" }
+    it { should parse "messenger <p>bag</p> skateboard" }
+    it { should parse "<p>messenger</p><p>bag</p><p>skateboard</p>" }
+    it { should parse "<p>messenger</p>\n<p>bag</p>\n<p>skateboard</p>" }
+    it { should parse "<p>messenger <strong>bag</strong> skateboard</p>" }
+  end
+
+  context "#element" do
+    subject { parser.element }
+
+    it { should     parse "<p></p>" }
+    it { should     parse "<p>messenger bag skateboard</p>" }
+    it { should     parse %q{<tofu art="party" />} }
+    it { should_not parse "<p>" }
+    it { should_not parse "<p>messenger bag skateboard" }
+    it { should_not parse "messenger bag skateboard</p>" }
+    it { should_not parse "<p>messenger bag skateboard<p>" }
+  end
+
+  context "#text" do
+    subject { parser.text }
+
+    it { should     parse "messenger bag skateboard" }
+    it { should_not parse "<p>messenger bag skateboard</p>" }
+    it { should_not parse "" }
+  end
+
+  context "#start_tag" do
+    subject { parser.start_tag }
+
+    it { should     parse %q{<tofu art="party">} }
+    it { should     parse %q{<tofu art="party" synth="letterpress">} }
+    it { should     parse "<tofu>" }
+    it { should_not parse "</tofu>" }
+    it { should_not parse "<tofu" }
+    it { should_not parse "tofu>" }
+  end
+
+  context "#end_tag" do
+    subject { parser.end_tag }
+
+    it { should     parse "</tofu>" }
+    it { should_not parse "<tofu>" }
+    it { should_not parse "<tofu" }
+    it { should_not parse "/tofu>" }
+  end
+
+  context "#empty_tag" do
+    subject { parser.empty_tag }
+
+    it { should     parse %q{<tofu art="party" />} }
+    it { should     parse %q{<tofu art="party" synth="letterpress" />} }
+    it { should_not parse "<tofu>" }
+    it { should_not parse "</tofu>" }
+    it { should_not parse "<tofu" }
+    it { should_not parse "/tofu>" }
+  end
+
+  context "#name" do
+    subject { parser.name }
+
+    it { should     parse "p" }
+    it { should     parse "h1" }
+    it { should_not parse "1h" }
+    it { should_not parse "h 1" }
+  end
+
+  context "#attribute" do
+    subject { parser.attribute }
+
+    it { should     parse %q{art="party organic"} }
+    it { should     parse %q{art='party organic'} }
+    it { should_not parse "art" }
+    it { should_not parse "art=" }
+    it { should_not parse "art=party" }
+    it { should_not parse %q{="party organic"} }
+    it { should_not parse %q{art="party organic'} }
+    it { should_not parse %q{art='party organic"} }
+  end
+
+  context "#parse" do
+    subject { parser.parse(html) }
+
+    context "single element" do
+      let(:html) { "<p>messenger</p>" }
+
+      it do
+        should == [
+          {
+            element: {
+              start_tag: {name: "p", attributes: []},
+              end_tag:   {name: "p"},
+              content:   [{text: "messenger"}]
+            }
+          }
+        ]
+      end
+    end
+
+    context "single element with attributes" do
+      let(:html) { %q{<a href="http://helvetica.com/" title="art party organic">messenger bag skateboard</a>} }
+
+      it do
+        should == [
+          {
+            element: {
+              start_tag: {
+                name: "a",
+                attributes: [
+                  {name: "href",  value: "http://helvetica.com/"},
+                  {name: "title", value: "art party organic"}
+                ]
+              },
+              end_tag: {name: "a"},
+              content: [{text: "messenger bag skateboard"}]
+            }
+          }
+        ]
+      end
+    end
+
+    context "multiple inline elements" do
+      let(:html) { "<p>messenger</p><p>bag</p><p>skateboard</p>" }
+
+      it do
+        should == [
+          {
+            element: {
+              start_tag: {name: "p", attributes: []},
+              end_tag:   {name: "p"},
+              content:   [{text: "messenger"}]
+            }
+          }, {
+            element: {
+              start_tag: {name: "p", attributes: []},
+              end_tag:   {name: "p"},
+              content:   [{text: "bag"}]
+            }
+          }, {
+            element: {
+              start_tag: {name: "p", attributes: []},
+              end_tag:   {name: "p"},
+              content:   [{text: "skateboard"}]
+            }
+          }
+        ]
+      end
+    end
+
+    context "multiple elements" do
+      let(:html) { "<p>messenger</p>\n<p>bag</p>\n<p>skateboard</p>" }
+
+      it do
+        should == [
+          {
+            element: {
+              start_tag: {name: "p", attributes: []},
+              end_tag:   {name: "p"},
+              content:   [{text: "messenger"}]
+            }
+          }, {
+            text: "\n"
+          }, {
+            element: {
+              start_tag: {name: "p", attributes: []},
+              end_tag:   {name: "p"},
+              content:   [{text: "bag"}]
+            }
+          }, {
+            text: "\n"
+          }, {
+            element: {
+              start_tag: {name: "p", attributes: []},
+              end_tag:   {name: "p"},
+              content:   [{text: "skateboard"}]
+            }
+          }
+        ]
+      end
+    end
+
+    context "nested elements" do
+      let(:html) { "<p>messenger <strong>bag</strong> skateboard</p>" }
+
+      it do
+        should == [
+          {
+            element: {
+              start_tag: {name: "p", attributes: []},
+              end_tag:   {name: "p"},
+              content:   [
+                {text: "messenger "},
+                {
+                  element: {
+                    start_tag: {name: "strong", attributes: []},
+                    content:   [{text: "bag"}],
+                    end_tag:   {name: "strong"}
+                  }
+                }, {
+                  text: " skateboard"
+                }
+              ]
+            }
+          }
+        ]
+      end
+    end
+  end
+end

data/spec/unit/lib/upmark/transform/markdown_spec.rb

@@ -0,0 +1,98 @@
+require "spec_helper"
+
+describe Upmark::Transform::Markdown do
+  let(:transform) { Upmark::Transform::Markdown.new }
+
+  context "#apply" do
+    subject { transform.apply(ast) }
+
+    context "<p>" do
+      context "single element" do
+        let(:ast) do
+          [
+            {
+              element: {
+                tag:     {name: "p", attributes: []},
+                content: [{text: "messenger bag skateboard"}]
+              }
+            }
+          ]
+        end
+
+        it { should == ["messenger bag skateboard\n\n"] }
+      end
+
+      context "multiple elements" do
+        let(:ast) do
+          [
+            {
+              element: {
+                tag:     {name: "p", attributes: []},
+                content: [{text: "messenger"}]
+              }
+            }, {
+              element: {
+                tag:     {name: "p", attributes: []},
+                content: [{text: "bag"}]
+              }
+            }, {
+              element: {
+                tag:     {name: "p", attributes: []},
+                content: [{text: "skateboard"}]
+              }
+            }
+          ]
+        end
+
+        it { should == ["messenger\n\n", "bag\n\n", "skateboard\n\n"] }
+      end
+    end
+
+    context "<a>" do
+      context "single element" do
+        let(:ast) do
+          [
+            {
+              element: {
+                tag: {
+                  name: "a",
+                  attributes: [
+                    {name: "href",  value: "http://helvetica.com/"},
+                    {name: "title", value: "art party organic"}
+                  ]
+                },
+                content: [{text: "messenger bag skateboard"}]
+              }
+            }
+          ]
+        end
+
+        it { should == [%q{[messenger bag skateboard](http://helvetica.com/ "art party organic")}] }
+      end
+    end
+
+    context "<img>" do
+      context "empty element" do
+        let(:ast) do
+          [
+            {
+              element: {
+                tag: {
+                  name: "img",
+                  attributes: [
+                    {name: "src",   value: "http://helvetica.com/image.gif"},
+                    {name: "title", value: "art party organic"},
+                    {name: "alt",   value: "messenger bag skateboard"}
+                  ]
+                },
+                content: []
+              }
+            }
+          ]
+        end
+
+        it { should == [%q{![messenger bag skateboard](http://helvetica.com/image.gif "art party organic")}] }
+      end
+    end
+  end
+end

data/upmark.gemspec

@@ -0,0 +1,26 @@
+# encoding: utf-8
+
+$:.push File.expand_path("../lib", __FILE__)
+
+require "upmark/version"
+
+Gem::Specification.new do |s|
+  s.name        = "upmark"
+  s.version     = Upmark::VERSION
+  s.authors     = ["Josh Bassett", "Gus Gollings"]
+  s.email       = "dev@theconversation.edu.au"
+  s.homepage    = "https://github.com/conversation/upmark"
+  s.summary     = %q{A HTML to Markdown converter.}
+  s.description = %q{Upmark has the skills to convert your HTML to Markdown.}
+
+  s.rubyforge_project = "upmark"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map {|f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rspec"
+
+  s.add_runtime_dependency "parslet"
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: upmark
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Josh Bassett
+- Gus Gollings
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-09-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70160857809560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70160857809560
+- !ruby/object:Gem::Dependency
+  name: parslet
+  requirement: &70160857808520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70160857808520
+description: Upmark has the skills to convert your HTML to Markdown.
+email: dev@theconversation.edu.au
+executables:
+- upmark
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.md
+- README.md
+- Rakefile
+- bin/upmark
+- lib/core_ext/array.rb
+- lib/upmark.rb
+- lib/upmark/parser/xml.rb
+- lib/upmark/transform/markdown.rb
+- lib/upmark/transform/preprocess.rb
+- lib/upmark/version.rb
+- spec/acceptance/upmark_spec.rb
+- spec/spec_helper.rb
+- spec/unit/lib/upmark/parser/xml_spec.rb
+- spec/unit/lib/upmark/transform/markdown_spec.rb
+- upmark.gemspec
+homepage: https://github.com/conversation/upmark
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: upmark
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: A HTML to Markdown converter.
+test_files:
+- spec/acceptance/upmark_spec.rb
+- spec/spec_helper.rb
+- spec/unit/lib/upmark/parser/xml_spec.rb
+- spec/unit/lib/upmark/transform/markdown_spec.rb