gitlab-markup 1.5.0.pre

data/lib/github/markup.rb

@@ -0,0 +1,55 @@
+require "github/markup/command_implementation"
+require "github/markup/gem_implementation"
+
+module GitHub
+  module Markup
+    extend self
+    @@markups = []
+
+    def markups
+      @@markups
+    end
+
+    def preload!
+      markups.each do |markup|
+        markup.load
+      end
+    end
+
+    def render(filename, content = nil)
+      content ||= File.read(filename)
+
+      if impl = renderer(filename)
+        impl.render(content)
+      else
+        content
+      end
+    end
+
+    def markup(file, pattern, opts = {}, &block)
+      markups << GemImplementation.new(pattern, file, &block)
+    end
+
+    def command(command, regexp, name, &block)
+      if File.exist?(file = File.dirname(__FILE__) + "/commands/#{command}")
+        command = file
+      end
+
+      markups << CommandImplementation.new(regexp, command, name, &block)
+    end
+
+    def can_render?(filename)
+      !!renderer(filename)
+    end
+
+    def renderer(filename)
+      markups.find { |impl|
+        impl.match?(filename)
+      }
+    end
+
+    # Define markups
+    markups_rb = File.dirname(__FILE__) + '/markups.rb'
+    instance_eval File.read(markups_rb), markups_rb
+  end
+end

data/lib/github/markup/command_implementation.rb

@@ -0,0 +1,71 @@
+begin
+  require "posix-spawn"
+rescue LoadError
+  require "open3"
+end
+
+require "github/markup/implementation"
+
+module GitHub
+  module Markup
+    class CommandError < RuntimeError
+    end
+
+    class CommandImplementation < Implementation
+      attr_reader :command, :block, :name
+
+      def initialize(regexp, command, name, &block)
+        super regexp
+        @command = command.to_s
+        @block = block
+        @name = name
+      end
+
+      def render(content)
+        rendered = execute(command, content)
+        rendered = rendered.to_s.empty? ? content : rendered
+        call_block(rendered, content)
+      end
+
+    private
+      def call_block(rendered, content)
+        if block && block.arity == 2
+          block.call(rendered, content)
+        elsif block
+          block.call(rendered)
+        else
+          rendered
+        end
+      end
+
+      if defined?(POSIX::Spawn)
+        def execute(command, target)
+          spawn = POSIX::Spawn::Child.new(*command, :input => target)
+          if spawn.status.success?
+            sanitize(spawn.out, target.encoding)
+          else
+            raise CommandError.new(spawn.err.strip)
+          end
+        end
+      else
+        def execute(command, target)
+          output = Open3.popen3(*command) do |stdin, stdout, stderr, wait_thr|
+            stdin.puts target
+            stdin.close
+            if wait_thr.value.success?
+              stdout.readlines
+            else
+              raise CommandError.new(stderr.readlines.join('').strip)
+            end
+          end
+          sanitize(output.join(''), target.encoding)
+        end
+      end
+
+      def sanitize(input, encoding)
+        input.gsub("\r", '').force_encoding(encoding)
+      end
+
+    end
+  end
+end

data/lib/github/markup/gem_implementation.rb

@@ -0,0 +1,30 @@
+require "github/markup/implementation"
+
+module GitHub
+  module Markup
+    class GemImplementation < Implementation
+      attr_reader :gem_name, :renderer
+
+      def initialize(regexp, gem_name, &renderer)
+        super regexp
+        @gem_name = gem_name.to_s
+        @renderer = renderer
+      end
+
+      def load
+        return if @loaded
+        require gem_name
+        @loaded = true
+      end
+
+      def render(content)
+        load
+        renderer.call(content)
+      end
+
+      def name
+        gem_name
+      end
+    end
+  end
+end

data/lib/github/markup/implementation.rb

@@ -0,0 +1,28 @@
+module GitHub
+  module Markup
+    class Implementation
+      attr_reader :regexp
+
+      def initialize(regexp)
+        @regexp = regexp
+      end
+
+      def load
+        # no-op by default
+      end
+
+      def render(content)
+        raise NotImplementedError, "subclasses of GitHub::Markup::Implementation must define #render"
+      end
+
+      def match?(filename)
+        file_ext_regexp =~ filename
+      end
+
+    private
+      def file_ext_regexp
+        @file_ext_regexp ||= /\.(#{regexp})\z/
+      end
+    end
+  end
+end

data/lib/github/markup/markdown.rb

@@ -0,0 +1,60 @@
+require "github/markup/implementation"
+
+module GitHub
+  module Markup
+    class Markdown < Implementation
+      MARKDOWN_GEMS = {
+        "github/markdown" => proc { |content|
+          GitHub::Markdown.render(content)
+        },
+        "redcarpet" => proc { |content|
+          Redcarpet::Markdown.new(Redcarpet::Render::HTML).render(content)
+        },
+        "rdiscount" => proc { |content|
+          RDiscount.new(content).to_html
+        },
+        "maruku" => proc { |content|
+          Maruku.new(content).to_html
+        },
+        "kramdown" => proc { |content|
+          Kramdown::Document.new(content).to_html
+        },
+        "bluecloth" => proc { |content|
+          BlueCloth.new(content).to_html
+        },
+      }
+
+      def initialize
+        super(/md|rmd|mkdn?|mdwn|mdown|markdown|litcoffee/i)
+      end
+
+      def load
+        return if @renderer
+        MARKDOWN_GEMS.each do |gem_name, renderer|
+          if try_require(gem_name)
+            @renderer = renderer
+            return
+          end
+        end
+        raise LoadError, "no suitable markdown gem found"
+      end
+
+      def render(content)
+        load
+        @renderer.call(content)
+      end
+
+      def name
+        "markdown"
+      end
+
+    private
+      def try_require(file)
+        require file
+        true
+      rescue LoadError
+        false
+      end
+    end
+  end
+end

data/lib/github/markup/rdoc.rb

@@ -0,0 +1,26 @@
+require "github/markup/implementation"
+require "rdoc"
+require "rdoc/markup/to_html"
+
+module GitHub
+  module Markup
+    class RDoc < Implementation
+      def initialize
+        super /rdoc/
+      end
+
+      def render(content)
+        if ::RDoc::VERSION.to_i >= 4
+          h = ::RDoc::Markup::ToHtml.new(::RDoc::Options.new)
+        else
+          h = ::RDoc::Markup::ToHtml.new
+        end
+        h.convert(content)
+      end
+
+      def name
+        "rdoc"
+      end
+    end
+  end
+end

data/lib/github/markups.rb

@@ -0,0 +1,50 @@
+require "github/markup/markdown"
+require "github/markup/rdoc"
+require "shellwords"
+
+markups << GitHub::Markup::Markdown.new
+
+markup(:redcloth, /textile/) do |content|
+  RedCloth.new(content).to_html
+end
+
+markups << GitHub::Markup::RDoc.new
+
+markup('org-ruby', /org/) do |content|
+  Orgmode::Parser.new(content, {
+                        :allow_include_files => false,
+                        :skip_syntax_highlight => true
+                      }).to_html
+end
+
+markup(:creole, /creole/) do |content|
+  Creole.creolize(content)
+end
+
+markup(:wikicloth, /mediawiki|wiki/) do |content|
+  wikicloth = WikiCloth::WikiCloth.new(:data => content)
+  WikiCloth::WikiBuffer::HTMLElement::ESCAPED_TAGS << 'tt'
+  wikicloth.to_html(:noedit => true)
+end
+
+markup(:asciidoctor, /adoc|asc(iidoc)?/) do |content|
+  Asciidoctor::Compliance.unique_id_start_index = 1
+  Asciidoctor.convert(content, :safe => :secure, :attributes => %w(showtitle=@ idprefix idseparator=- env=github env-github source-highlighter=html-pipeline))
+end
+
+command(
+  "python2 -S #{Shellwords.escape(File.dirname(__FILE__))}/commands/rest2html",
+  /re?st(\.txt)?/,
+  "restructuredtext"
+)
+
+# pod2html is nice enough to generate a full-on HTML document for us,
+# so we return the favor by ripping out the good parts.
+#
+# Any block passed to `command` will be handed the command's STDOUT for
+# post processing.
+command('/usr/bin/env perl -MPod::Simple::HTML -e Pod::Simple::HTML::go', /pod/, "pod") do |rendered|
+  if rendered =~ /<!-- start doc -->\s*(.+)\s*<!-- end doc -->/mi
+    $1
+  end
+end

data/script/bootstrap

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+cd $(dirname "$0")/..
+
+bundle install
+easy_install docutils

data/script/cibuild

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+# GC customizations
+export RUBY_GC_MALLOC_LIMIT=79000000
+export RUBY_HEAP_MIN_SLOTS=800000
+export RUBY_HEAP_FREE_MIN=100000
+export RUBY_HEAP_SLOTS_INCREMENT=400000
+export RUBY_HEAP_SLOTS_GROWTH_FACTOR=1
+
+export PATH="/usr/share/rbenv/shims:$PATH"
+export RBENV_VERSION="1.9.3"
+
+# bootstrap gem environment changes
+echo "Bootstrapping gem environment ..."
+
+script/bootstrap --local
+
+rake

data/test/fixtures/fail.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "failure message">&2 && false

data/test/markup_test.rb

@@ -0,0 +1,116 @@
+# encoding: UTF-8
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'github/markup'
+require 'minitest/autorun'
+require 'html/pipeline'
+require 'nokogiri'
+require 'nokogiri/diff'
+
+def normalize_html(text)
+  text.strip!
+  text.gsub!(/\s\s+/,' ')
+  text.gsub!(/\p{Pi}|\p{Pf}|"/u,'"')
+  text.gsub!("\u2026",'...')
+  text
+end
+
+def assert_html_equal(expected, actual, msg = nil)
+    assertion = Proc.new do
+      expected_doc = Nokogiri::HTML(expected) {|config| config.noblanks}
+      actual_doc   = Nokogiri::HTML(actual) {|config| config.noblanks}
+
+      expected_doc.search('//text()').each {|node| node.content = normalize_html node.content}
+      actual_doc.search('//text()').each {|node| node.content = normalize_html node.content}
+
+      ignore_changes = {"+" => Regexp.union(/^\s*id=".*"\s*$/), "-" => nil}
+      expected_doc.diff(actual_doc) do |change, node|
+        if change != ' ' && !node.blank? then
+          break unless node.to_html =~ ignore_changes[change]
+        end
+      end
+    end
+    assert(assertion.call, msg)
+end
+
+class MarkupTest < Minitest::Test
+  class MarkupFilter < HTML::Pipeline::Filter
+    def call
+      filename = context[:filename]
+      GitHub::Markup.render(filename, File.read(filename)).strip.force_encoding("utf-8")
+    end
+  end
+
+  Pipeline = HTML::Pipeline.new [
+    MarkupFilter,
+    HTML::Pipeline::SanitizationFilter
+  ]
+
+  Dir['test/markups/README.*'].each do |readme|
+    next if readme =~ /html$/
+    markup = readme.split('/').last.gsub(/^README\./, '')
+
+    define_method "test_#{markup}" do
+      skip "Skipping MediaWiki test because wikicloth is currently not compatible with JRuby." if markup == "mediawiki" && RUBY_PLATFORM == "java"
+
+      source = File.read(readme)
+      expected_file = "#{readme}.html"
+      expected = File.read(expected_file).rstrip
+      actual = Pipeline.to_html(nil, :filename => readme)
+
+      if source != expected
+        assert(source != actual, "#{markup} did not render anything")
+      end
+
+      diff = IO.popen("diff -u - #{expected_file}", 'r+') do |f|
+        f.write actual
+        f.close_write
+        f.read
+      end
+
+      assert_html_equal expected, actual, <<message
+#{File.basename expected_file}'s contents are not html equal to output:
+#{diff}
+message
+    end
+  end
+
+  def test_knows_what_it_can_and_cannot_render
+    assert_equal false, GitHub::Markup.can_render?('README.html')
+    assert_equal true, GitHub::Markup.can_render?('README.markdown')
+    assert_equal true, GitHub::Markup.can_render?('README.rmd')
+    assert_equal true, GitHub::Markup.can_render?('README.Rmd')
+    assert_equal false, GitHub::Markup.can_render?('README.cmd')
+    assert_equal true, GitHub::Markup.can_render?('README.litcoffee')
+  end
+
+  def test_each_render_has_a_name
+    assert_equal "markdown", GitHub::Markup.renderer('README.md').name
+    assert_equal "redcloth", GitHub::Markup.renderer('README.textile').name
+    assert_equal "rdoc", GitHub::Markup.renderer('README.rdoc').name
+    assert_equal "org-ruby", GitHub::Markup.renderer('README.org').name
+    assert_equal "creole", GitHub::Markup.renderer('README.creole').name
+    assert_equal "wikicloth", GitHub::Markup.renderer('README.wiki').name
+    assert_equal "asciidoctor", GitHub::Markup.renderer('README.adoc').name
+    assert_equal "restructuredtext", GitHub::Markup.renderer('README.rst').name
+    assert_equal "pod", GitHub::Markup.renderer('README.pod').name
+  end
+
+  def test_raises_error_if_command_exits_non_zero
+    GitHub::Markup.command('test/fixtures/fail.sh', /fail/, 'fail')
+    assert GitHub::Markup.can_render?('README.fail')
+    begin
+      GitHub::Markup.render('README.fail', "stop swallowing errors")
+    rescue GitHub::Markup::CommandError => e
+      assert_equal "failure message", e.message
+    else
+      fail "an exception was expected but was not raised"
+    end
+  end
+
+  def test_preserve_markup
+    content = "Noël"
+    assert_equal content.encoding.name, GitHub::Markup.render('Foo.rst', content).encoding.name
+  end
+end