qiita_marker 0.23.2.0

data/lib/qiita_marker/renderer.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'stringio'
+
+module QiitaMarker
+  class Renderer
+    attr_accessor :in_tight, :warnings, :in_plain
+
+    def initialize(options: :DEFAULT, extensions: [])
+      @opts = Config.process_options(options, :render)
+      @stream = StringIO.new(+'')
+      @need_blocksep = false
+      @warnings = Set.new []
+      @in_tight = false
+      @in_plain = false
+      @tagfilter = extensions.include?(:tagfilter)
+    end
+
+    def out(*args)
+      args.each do |arg|
+        case arg
+        when :children
+          @node.each { |child| out(child) }
+        when Array
+          arg.each { |x| render(x) }
+        when Node
+          render(arg)
+        else
+          @stream.write(arg)
+        end
+      end
+    end
+
+    def render(node)
+      @node = node
+      if node.type == :document
+        document(node)
+        @stream.string
+      elsif @in_plain && node.type != :text && node.type != :softbreak
+        node.each { |child| render(child) }
+      else
+        begin
+          send(node.type, node)
+        rescue NoMethodError => e
+          @warnings.add("WARNING: #{node.type} not implemented.")
+          raise e
+        end
+      end
+    end
+
+    def document(_node)
+      out(:children)
+    end
+
+    def code_block(node)
+      code_block(node)
+    end
+
+    def reference_def(_node); end
+
+    def cr
+      return if @stream.string.empty? || @stream.string[-1] == "\n"
+
+      out("\n")
+    end
+
+    def blocksep
+      out("\n")
+    end
+
+    def containersep
+      cr unless @in_tight
+    end
+
+    def block
+      cr
+      yield
+      cr
+    end
+
+    def container(starter, ender)
+      out(starter)
+      yield
+      out(ender)
+    end
+
+    def plain
+      old_in_plain = @in_plain
+      @in_plain = true
+      yield
+      @in_plain = old_in_plain
+    end
+
+    private
+
+    def escape_href(str)
+      @node.html_escape_href(str)
+    end
+
+    def escape_html(str)
+      @node.html_escape_html(str)
+    end
+
+    def tagfilter(str)
+      if @tagfilter
+        str.gsub(
+          %r{
+            <
+            (
+            title|textarea|style|xmp|iframe|
+            noembed|noframes|script|plaintext
+            )
+            (?=\s|>|/>)
+          }xi,
+          '&lt;\1'
+        )
+      else
+        str
+      end
+    end
+
+    def sourcepos(node)
+      return '' unless option_enabled?(:SOURCEPOS)
+
+      s = node.sourcepos
+      " data-sourcepos=\"#{s[:start_line]}:#{s[:start_column]}-" \
+        "#{s[:end_line]}:#{s[:end_column]}\""
+    end
+
+    def option_enabled?(opt)
+      (@opts & QiitaMarker::Config::OPTS.dig(:render, opt)) != 0
+    end
+  end
+end

data/lib/qiita_marker/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module QiitaMarker
+  VERSION = '0.23.2.0'
+end

data/lib/qiita_marker.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'qiita_marker/qiita_marker'
+require 'qiita_marker/config'
+require 'qiita_marker/node'
+require 'qiita_marker/renderer'
+require 'qiita_marker/renderer/html_renderer'
+require 'qiita_marker/version'
+
+begin
+  require 'awesome_print'
+rescue LoadError; end # rubocop:disable Lint/SuppressedException
+module QiitaMarker
+  # Public:  Parses a Markdown string into an HTML string.
+  #
+  # text - A {String} of text
+  # option - Either a {Symbol} or {Array of Symbol}s indicating the render options
+  # extensions - An {Array of Symbol}s indicating the extensions to use
+  #
+  # Returns a {String} of converted HTML.
+  def self.render_html(text, options = :DEFAULT, extensions = [])
+    raise TypeError, "text must be a String; got a #{text.class}!" unless text.is_a?(String)
+
+    opts = Config.process_options(options, :render)
+    text = text.encode('UTF-8')
+    html = Node.markdown_to_html(text, opts, extensions)
+    html.force_encoding('UTF-8')
+  end
+
+  # Public: Parses a Markdown string into a `document` node.
+  #
+  # string - {String} to be parsed
+  # option - A {Symbol} or {Array of Symbol}s indicating the parse options
+  # extensions - An {Array of Symbol}s indicating the extensions to use
+  #
+  # Returns the `document` node.
+  def self.render_doc(text, options = :DEFAULT, extensions = [])
+    raise TypeError, "text must be a String; got a #{text.class}!" unless text.is_a?(String)
+
+    opts = Config.process_options(options, :parse)
+    text = text.encode('UTF-8')
+    Node.parse_document(text, text.bytesize, opts, extensions)
+  end
+end

data/qiita_marker.gemspec

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'qiita_marker/version'
+
+Gem::Specification.new do |s|
+  s.name = 'qiita_marker'
+  s.version = QiitaMarker::VERSION
+  s.summary = 'Qiita Marker is a Ruby library for Markdown processing, based on CommonMarker.'
+  s.description = 'A Ruby library that is the core module of the Qiita-specified markdown processor.'
+  s.authors = ['Increments Inc.']
+  s.homepage = 'https://github.com/increments/qiita_marker'
+  s.license = 'MIT'
+
+  s.files         = %w[LICENSE.txt README.md Rakefile qiita_marker.gemspec bin/qiita_marker]
+  s.files        += Dir.glob('lib/**/*.rb')
+  s.files        += Dir.glob('ext/qiita_marker/*.*')
+  s.test_files    = Dir.glob('test/**/*').reject { |f| f == 'test/benchinput.md' || f.start_with?('test/progit/') }
+  s.extensions    = ['ext/qiita_marker/extconf.rb']
+
+  s.executables = ['qiita_marker']
+  s.require_paths = %w[lib ext]
+  s.required_ruby_version = ['>= 2.6', '< 4.0']
+
+  s.rdoc_options += ['-x', 'ext/qiita_marker/cmark/.*']
+
+  s.add_development_dependency 'awesome_print'
+  s.add_development_dependency 'json', '~> 2.3'
+  s.add_development_dependency 'minitest', '~> 5.6'
+  s.add_development_dependency 'minitest-focus', '~> 1.1'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rake-compiler', '~> 0.9'
+  s.add_development_dependency 'rdoc', '~> 6.2'
+  s.add_development_dependency 'rubocop'
+  s.add_development_dependency 'rubocop-standard'
+  s.metadata = {
+    'rubygems_mfa_required' => 'true'
+  }
+end

data/test/benchmark.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'qiita_marker'
+require 'github/markdown'
+require 'redcarpet'
+require 'kramdown'
+require 'benchmark'
+
+def dobench(name, &blk)
+  puts name
+  puts Benchmark.measure(&blk)
+end
+
+benchinput = File.open('test/benchinput.md', 'r').read
+
+printf("input size = %<bytes>d bytes\n\n", benchinput.bytesize)
+
+dobench('redcarpet') do
+  Redcarpet::Markdown.new(Redcarpet::Render::HTML, autolink: false, tables: false).render(benchinput)
+end
+
+dobench('qiita_marker with to_html') do
+  QiitaMarker.render_html(benchinput)
+end
+
+dobench('qiita_marker with ruby HtmlRenderer') do
+  QiitaMarker::HtmlRenderer.new.render(QiitaMarker.render_doc(benchinput))
+end
+
+dobench('kramdown') do
+  Kramdown::Document.new(benchinput).to_html(benchinput)
+end

data/test/fixtures/curly.md

@@ -0,0 +1 @@
+This curly quote “makes qiita_marker throw an exception”.

data/test/fixtures/dingus.md

@@ -0,0 +1,10 @@
+## Try CommonMark
+
+You can try CommonMark here.  This dingus is powered by
+[commonmark.js](https://github.com/jgm/commonmark.js), the
+JavaScript reference implementation.
+
+1. item one
+2. item two
+   - sublist
+   - sublist

data/test/fixtures/strong.md

@@ -0,0 +1 @@
+I am **strong**

data/test/fixtures/table.md

@@ -0,0 +1,10 @@
+One extension:
+
+| a   | b   |
+| --- | --- |
+| c   | d   |
+| **x** | |
+
+Another extension:
+
+~~hi~~

data/test/test_attributes.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class TestAttributes < Minitest::Test
+  def setup
+    contents = fixtures_file('dingus.md')
+    @doc = QiitaMarker.render_doc(contents.strip)
+  end
+
+  def test_sourcepos
+    sourcepos = []
+
+    @doc.walk do |node|
+      sourcepos << node.sourcepos
+    end
+
+    sourcepos.delete_if { |h| h.values.all?(&:zero?) }
+
+    result = [{ start_line: 1, start_column: 1, end_line: 10, end_column: 12 }, { start_line: 1, start_column: 1, end_line: 1, end_column: 17 }, { start_line: 1, start_column: 4, end_line: 1, end_column: 17 }, { start_line: 3, start_column: 1, end_line: 5, end_column: 36 }, { start_line: 3, start_column: 1, end_line: 3, end_column: 55 }, { start_line: 4, start_column: 1, end_line: 4, end_column: 53 }, { start_line: 4, start_column: 2, end_line: 4, end_column: 14 }, { start_line: 4, start_column: 54, end_line: 4, end_column: 58 }, { start_line: 5, start_column: 1, end_line: 5, end_column: 36 }, { start_line: 7, start_column: 1, end_line: 10, end_column: 12 }, { start_line: 7, start_column: 1, end_line: 7, end_column: 11 }, { start_line: 7, start_column: 4, end_line: 7, end_column: 11 }, { start_line: 7, start_column: 4, end_line: 7, end_column: 11 }, { start_line: 8, start_column: 1, end_line: 10, end_column: 12 }, { start_line: 8, start_column: 4, end_line: 8, end_column: 11 }, { start_line: 8, start_column: 4, end_line: 8, end_column: 11 }, { start_line: 9, start_column: 4, end_line: 10, end_column: 12 }, { start_line: 9, start_column: 4, end_line: 9, end_column: 12 }, { start_line: 9, start_column: 6, end_line: 9, end_column: 12 }, { start_line: 9, start_column: 6, end_line: 9, end_column: 12 }, { start_line: 10, start_column: 4, end_line: 10, end_column: 12 }, { start_line: 10, start_column: 6, end_line: 10, end_column: 12 }, { start_line: 10, start_column: 6, end_line: 10, end_column: 12 }]
+
+    assert_equal result, sourcepos
+  end
+end

data/test/test_basics.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class TestBasics < Minitest::Test
+  def setup
+    @doc = QiitaMarker.render_doc('Hi *there*')
+  end
+
+  def test_to_html
+    assert_equal "<p>Hi <em>there</em></p>\n", @doc.to_html
+  end
+
+  def test_markdown_to_html
+    html = QiitaMarker.render_html('Hi *there*')
+    assert_equal "<p>Hi <em>there</em></p>\n", html
+  end
+
+  # basic test that just checks if every option is accepted & no errors are thrown
+  def test_accept_every_option
+    text = "Hello **world** -- how are _you_ today? I'm ~~fine~~, ~yourself~?"
+    parse_opt = %i[SOURCEPOS UNSAFE VALIDATE_UTF8 SMART LIBERAL_HTML_TAG FOOTNOTES STRIKETHROUGH_DOUBLE_TILDE]
+    render_opt = parse_opt + %i[HARDBREAKS NOBREAKS GITHUB_PRE_LANG TABLE_PREFER_STYLE_ATTRIBUTES FULL_INFO_STRING]
+
+    extensions = %i[table tasklist strikethrough autolink tagfilter]
+
+    assert_equal "<p>Hello <strong>world</strong> – how are <em>you</em> today? I’m <del>fine</del>, ~yourself~?</p>\n", QiitaMarker.render_doc(text, parse_opt, extensions).to_html
+
+    # NOTE: how tho the doc returned has sourcepos info, by default the renderer
+    # won't emit it. for that we need to pass in the render opt
+    assert_equal "<p data-sourcepos=\"1:1-1:65\">Hello <strong>world</strong> – how are <em>you</em> today? I’m <del>fine</del>, ~yourself~?</p>\n", QiitaMarker.render_doc(text, parse_opt, extensions).to_html(render_opt, extensions)
+
+    assert_equal "<p data-sourcepos=\"1:1-1:65\">Hello <strong>world</strong> – how are <em>you</em> today? I’m <del>fine</del>, ~yourself~?</p>\n", QiitaMarker.render_html(text, parse_opt, extensions)
+  end
+end

data/test/test_commands.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class TestCommands < Minitest::Test
+  def test_basic
+    out = make_bin('strong.md')
+    assert_equal('<p>I am <strong>strong</strong></p>', out)
+  end
+
+  def test_does_not_have_extensions
+    out = make_bin('table.md')
+    assert_includes out, '| a'
+    refute_includes out, '<p><del>hi</del>'
+    refute_includes out, '<table> <tr> <th> a </th> <td> c </td>'
+  end
+
+  def test_understands_extensions
+    out = make_bin('table.md', '--extension=table')
+    refute_includes out, '| a'
+    refute_includes out, '<p><del>hi</del>'
+    %w[<table> <tr> <th> a </th> <td> c </td>].each { |html| assert_includes out, html }
+  end
+
+  def test_understands_multiple_extensions
+    out = make_bin('table.md', '--extension=table,strikethrough')
+    refute_includes out, '| a'
+    assert_includes out, '<p><del>hi</del>'
+    %w[<table> <tr> <th> a </th> <td> c </td>].each { |html| assert_includes out, html }
+  end
+
+  def test_understands_html_format_with_renderer_and_extensions
+    out = make_bin('table.md', '--to=html --extension=table,strikethrough --html-renderer')
+    refute_includes out, '| a'
+    assert_includes out, '<p><del>hi</del>'
+    %w[<table> <tr> <th> a </th> <td> c </td>].each { |html| assert_includes out, html }
+  end
+
+  def test_understands_xml_format
+    out = make_bin('strong.md', '--to=xml')
+    assert_includes out, '<?xml version="1.0" encoding="UTF-8"?>'
+    assert_includes out, '<text xml:space="preserve">strong</text>'
+  end
+
+  def test_understands_commonmark_format
+    out = make_bin('strong.md', '--to=commonmark')
+    assert_equal('I am **strong**', out)
+  end
+
+  def test_understands_plaintext_format
+    out = make_bin('strong.md', '--to=plaintext')
+    assert_equal('I am strong', out)
+  end
+
+  def test_aborts_invalid_format
+    _out, err = capture_subprocess_io do
+      make_bin('strong.md', '--to=unknown')
+    end
+
+    assert_match "format 'unknown' not found", err
+  end
+
+  def test_aborts_format_and_html_renderer_combinations
+    (QiitaMarker::Config::OPTS[:format] - [:html]).each do |format|
+      _out, err = capture_subprocess_io do
+        make_bin('strong.md', "--to=#{format} --html-renderer")
+      end
+
+      assert_match "format '#{format}' does not support using the HtmlRenderer renderer", err
+    end
+  end
+end

data/test/test_commonmark.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class TestCommonmark < Minitest::Test
+  HTML_COMMENT = /<!--.*?-->\s?/.freeze
+
+  def setup
+    @markdown = <<~MD
+      Hi *there*!
+
+      1. I am a numeric list.
+      2. I continue the list.
+      * Suddenly, an unordered list!
+      * What fun!
+
+      Okay, _enough_.
+
+      | a   | b   |
+      | --- | --- |
+      | c   | d   |
+    MD
+  end
+
+  def render_doc(doc)
+    QiitaMarker.render_doc(doc, :DEFAULT, %i[table])
+  end
+
+  def test_to_commonmark
+    compare = render_doc(@markdown).to_commonmark
+
+    assert_equal \
+      render_doc(@markdown).to_html.squeeze(' ').gsub(HTML_COMMENT, ''),
+      render_doc(compare).to_html.squeeze(' ').gsub(HTML_COMMENT, '')
+  end
+end

data/test/test_doc.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class TestDocNode < Minitest::Test
+  def setup
+    @doc = QiitaMarker.render_doc('Hi *there*. This has __many nodes__!')
+    @first_child = @doc.first_child
+    @last_child = @doc.last_child
+    @link = QiitaMarker.render_doc('[GitHub](https://www.github.com)').first_child.first_child
+    @image = QiitaMarker.render_doc('![alt text](https://github.com/favicon.ico "Favicon")')
+    @image = @image.first_child.first_child
+    @header = QiitaMarker.render_doc('### Header Three').first_child
+    @ul_list = QiitaMarker.render_doc("* Bullet\n*Bullet").first_child
+    @ol_list = QiitaMarker.render_doc("1. One\n2. Two").first_child
+    @fence = QiitaMarker.render_doc("``` ruby\nputs 'wow'\n```").first_child
+  end
+
+  def test_get_type
+    assert_equal(:document, @doc.type)
+  end
+
+  def test_get_type_string
+    assert_equal('document', @doc.type_string)
+  end
+
+  def test_get_first_child
+    assert_equal(:paragraph, @first_child.type)
+  end
+
+  def test_get_next
+    assert_equal(:emph, @first_child.first_child.next.type)
+  end
+
+  def test_insert_before
+    paragraph = Node.new(:paragraph)
+    assert(@first_child.insert_before(paragraph))
+    assert_match "<p></p>\n<p>Hi <em>there</em>.", @doc.to_html
+  end
+
+  def test_insert_after
+    paragraph = Node.new(:paragraph)
+    assert(@first_child.insert_after(paragraph))
+    assert_match "<strong>many nodes</strong>!</p>\n<p></p>\n", @doc.to_html
+  end
+
+  def test_prepend_child
+    code = Node.new(:code)
+    assert(@first_child.prepend_child(code))
+    assert_match '<p><code></code>Hi <em>there</em>.', @doc.to_html
+  end
+
+  def test_append_child
+    strong = Node.new(:strong)
+    assert(@first_child.append_child(strong))
+    assert_match "!<strong></strong></p>\n", @doc.to_html
+  end
+
+  def test_get_last_child
+    assert_equal(:paragraph, @last_child.type)
+  end
+
+  def test_get_parent
+    assert_equal(:paragraph, @first_child.first_child.next.parent.type)
+  end
+
+  def test_get_previous
+    assert_equal(:text, @first_child.first_child.next.previous.type)
+  end
+
+  def test_get_url
+    assert_equal('https://www.github.com', @link.url)
+  end
+
+  def test_set_url
+    assert_equal('https://www.mozilla.org', @link.url = 'https://www.mozilla.org')
+  end
+
+  def test_get_title
+    assert_equal('Favicon', @image.title)
+  end
+
+  def test_set_title
+    assert_equal('Octocat', @image.title = 'Octocat')
+  end
+
+  def test_get_header_level
+    assert_equal(3, @header.header_level)
+  end
+
+  def test_set_header_level
+    assert_equal(6, @header.header_level = 6)
+  end
+
+  def test_get_list_type
+    assert_equal(:bullet_list, @ul_list.list_type)
+    assert_equal(:ordered_list, @ol_list.list_type)
+  end
+
+  def test_set_list_type
+    assert_equal(:ordered_list, @ul_list.list_type = :ordered_list)
+    assert_equal(:bullet_list, @ol_list.list_type = :bullet_list)
+  end
+
+  def test_get_list_start
+    assert_equal(1, @ol_list.list_start)
+  end
+
+  def test_set_list_start
+    assert_equal(8, @ol_list.list_start = 8)
+  end
+
+  def test_get_list_tight
+    assert(@ul_list.list_tight)
+    assert(@ol_list.list_tight)
+  end
+
+  def test_set_list_tight
+    refute(@ul_list.list_tight = false)
+    refute(@ol_list.list_tight = false)
+  end
+
+  def test_get_fence_info
+    assert_equal('ruby', @fence.fence_info)
+  end
+
+  def test_set_fence_info
+    assert_equal('javascript', @fence.fence_info = 'javascript')
+  end
+end

data/test/test_encoding.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class TestEncoding < Minitest::Test
+  # see http://git.io/vq4FR
+  def test_encoding
+    contents = fixtures_file('curly.md')
+    doc = QiitaMarker.render_doc(contents, :SMART)
+    render = doc.to_html
+    assert_equal('<p>This curly quote “makes qiita_marker throw an exception”.</p>', render.rstrip)
+
+    render = doc.to_xml
+    assert_includes(render, '<text xml:space="preserve">This curly quote “makes qiita_marker throw an exception”.</text>')
+  end
+
+  def test_string_content_is_utf8
+    doc = QiitaMarker.render_doc('Hi *there*')
+    text = doc.first_child.last_child.first_child
+    assert_equal('there', text.string_content)
+    assert_equal('UTF-8', text.string_content.encoding.name)
+  end
+end