commonmarker 0.17.13 → 0.23.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6f11a40dfa6f3039bd560b2845c835d884f48351
-  data.tar.gz: 1137f49d8b7e1689e8b433e57f381617f51a97df
+SHA256:
+  metadata.gz: bc6d71b6e85fb61abc438252b07369d3264982cf8fb327d96e5e8333bb23841e
+  data.tar.gz: 1911965561ddf20104db09c44c4e5d25f65a277a28b825bc0c03bb172779a623
 SHA512:
-  metadata.gz: a58dee4e21d3b6b982f92075a3a9a6e5ac636ecd4babab6208074e8ff56b42b132ea99ef1f425368db14c4081bc53a0ac8d16c756463e0249041498c65f1faee
-  data.tar.gz: 7f5b3476d7359b9d03b5352b6660ef269f54968b76440a74c02ef2feb93b495264ae0cc1e90bde520ee12fd035ff92e094b64180b841efe6edebf01ba69083ab
+  metadata.gz: 59737cb433c3f4f3d53e06d3466f1f3dcd8a40e201e8b0f26be4d553f45667137e05d980100304e720d0e8fda178da3eaa9e421608cf4e53c6d159ff9261ae3e
+  data.tar.gz: c3088c3219da29bdb0356f46a5b7a7f2881410c825e101f7ced3964760bfcfcb5cef57e7f3ebf5bd9f686b974ad80887fc93899e616afb211dc0e648d6352cd2

data/README.md

@@ -1,6 +1,6 @@
 # CommonMarker
 
-[![Build Status](https://travis-ci.org/gjtorikian/commonmarker.svg)](https://travis-ci.org/gjtorikian/commonmarker) [![Gem Version](https://badge.fury.io/rb/commonmarker.svg)](http://badge.fury.io/rb/commonmarker)
+![Build Status](https://github.com/gjtorikian/commonmarker/workflows/CI/badge.svg) [![Gem Version](https://badge.fury.io/rb/commonmarker.svg)](http://badge.fury.io/rb/commonmarker)
 
 Ruby wrapper for [libcmark-gfm](https://github.com/github/cmark),
 GitHub's fork of the reference parser for CommonMark. It passes all of the C tests, and is therefore spec-complete. It also includes extensions to the CommonMark spec as documented in the [GitHub Flavored Markdown spec](http://github.github.com/gfm/), such as support for tables, strikethroughs, and autolinking.
@@ -112,10 +112,8 @@ class MyHtmlRenderer < CommonMarker::HtmlRenderer
   end
 end
 
-# this renderer prints directly to STDOUT, instead
-# of returning a string
 myrenderer = MyHtmlRenderer.new
-print(myrenderer.render(doc))
+puts myrenderer.render(doc)
 
 # Print any warnings to STDERR
 renderer.warnings.each do |w|
@@ -132,24 +130,31 @@ CommonMarker accepts the same options that CMark does, as symbols. Note that the
 | Name                          | Description
 | ----------------------------- | -----------
 | `:DEFAULT`                    | The default parsing system.
-| `:FOOTNOTES`                  | Parse footnotes.
-| `:LIBERAL_HTML_TAG`           | Support liberal parsing of inline HTML tags.
+| `:SOURCEPOS`                  | Include source position in nodes
+| `:UNSAFE`                     | Allow raw/custom HTML and unsafe links.
+| `:VALIDATE_UTF8`              | Replace illegal sequences with the replacement character `U+FFFD`.
 | `:SMART`                      | Use smart punctuation (curly quotes, etc.).
+| `:LIBERAL_HTML_TAG`           | Support liberal parsing of inline HTML tags.
+| `:FOOTNOTES`                  | Parse footnotes.
 | `:STRIKETHROUGH_DOUBLE_TILDE` | Parse strikethroughs by double tildes (compatibility with [redcarpet](https://github.com/vmg/redcarpet))
-| `:VALIDATE_UTF8`              | Replace illegal sequences with the replacement character `U+FFFD`.
 
 ### Render options
 
-| Name                             | Description                                                    |
-| ------------------               | -----------                                                    |
-| `:DEFAULT`                       | The default rendering system.                                  |
-| `:GITHUB_PRE_LANG`               | Use GitHub-style `<pre lang>` for fenced code blocks.          |
-| `:HARDBREAKS`                    | Treat `\n` as hardbreaks (by adding `<br/>`).                  |
-| `:NOBREAKS`                      | Translate `\n` in the source to a single whitespace.           |
-| `:SAFE`                          | Suppress raw HTML and unsafe links.                            |
-| `:SOURCEPOS`                     | Include source position in rendered HTML.                      |
-| `:TABLE_PREFER_STYLE_ATTRIBUTES` | Use `style` insted of `align` for table cells                  |
-| `:FULL_INFO_STRING`              | Include full info strings of code blocks in separate attribute |
+| Name                             | Description                                                     |
+| ------------------               | -----------                                                     |
+| `:DEFAULT`                       | The default rendering system.                                   |
+| `:SOURCEPOS`                     | Include source position in rendered HTML.                       |
+| `:HARDBREAKS`                    | Treat `\n` as hardbreaks (by adding `<br/>`).                   |
+| `:UNSAFE`                        | Allow raw/custom HTML and unsafe links.                         |
+| `:NOBREAKS`                      | Translate `\n` in the source to a single whitespace.            |
+| `:VALIDATE_UTF8`                 | Replace illegal sequences with the replacement character `U+FFFD`. |
+| `:SMART`                         | Use smart punctuation (curly quotes, etc.).                     |
+| `:GITHUB_PRE_LANG`               | Use GitHub-style `<pre lang>` for fenced code blocks.           |
+| `:LIBERAL_HTML_TAG`              | Support liberal parsing of inline HTML tags.                    |
+| `:FOOTNOTES`                     | Render footnotes.                                               |
+| `:STRIKETHROUGH_DOUBLE_TILDE`    | Parse strikethroughs by double tildes (compatibility with [redcarpet](https://github.com/vmg/redcarpet)) |
+| `:TABLE_PREFER_STYLE_ATTRIBUTES` | Use `style` insted of `align` for table cells.                  |
+| `:FULL_INFO_STRING`              | Include full info strings of code blocks in separate attribute. |
 
 ### Passing options
 
@@ -175,9 +180,80 @@ Both `render_html` and `render_doc` take an optional third argument defining the
 The available extensions are:
 
 * `:table` - This provides support for tables.
+* `:tasklist` - This provides support for task list items.
 * `:strikethrough` - This provides support for strikethroughs.
 * `:autolink` - This provides support for automatically converting URLs to anchor tags.
-* `:tagfilter` - This strips out [several "unsafe" HTML tags](https://github.github.com/gfm/#disallowed-raw-html-extension-) from being used.
+* `:tagfilter` - This escapes [several "unsafe" HTML tags](https://github.github.com/gfm/#disallowed-raw-html-extension-), causing them to not have any effect.
+
+## Output formats
+
+Like CMark, CommonMarker can generate output in several formats: HTML, XML, plaintext, and commonmark are currently supported.
+
+### HTML
+
+The default output format, HTML, will be generated when calling `to_html` or using `--to=html` on the command line.
+
+```ruby
+doc = CommonMarker.render_doc('*Hello* world!', :DEFAULT)
+puts(doc.to_html)
+
+<p><em>Hello</em> world!</p>
+```
+
+### XML
+
+XML will be generated when calling `to_xml` or using `--to=xml` on the command line.
+
+```ruby
+doc = CommonMarker.render_doc('*Hello* world!', :DEFAULT)
+puts(doc.to_xml)
+
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE document SYSTEM "CommonMark.dtd">
+<document xmlns="http://commonmark.org/xml/1.0">
+  <paragraph>
+    <emph>
+      <text xml:space="preserve">Hello</text>
+    </emph>
+    <text xml:space="preserve"> world!</text>
+  </paragraph>
+</document>
+```
+
+### Plaintext
+
+Plaintext will be generated when calling `to_plaintext` or using `--to=plaintext` on the command line.
+
+```ruby
+doc = CommonMarker.render_doc('*Hello* world!', :DEFAULT)
+puts(doc.to_plaintext)
+
+Hello world!
+```
+
+### Commonmark
+
+Commonmark will be generated when calling `to_commonmark` or using `--to=commonmark` on the command line.
+
+``` ruby
+text = <<-TEXT
+1. I am a numeric list.
+2. I continue the list.
+* Suddenly, an unordered list!
+* What fun!
+TEXT
+
+doc = CommonMarker.render_doc(text, :DEFAULT)
+puts(doc.to_commonmark)
+
+1.  I am a numeric list.
+2.  I continue the list.
+
+<!-- end list -->
+
+  - Suddenly, an unordered list\!
+  - What fun\!
+```
 
 ## Developing locally
 

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'date'
 require 'rake/clean'
 require 'rake/extensiontask'
@@ -6,7 +8,7 @@ require 'digest/md5'
 host_os = RbConfig::CONFIG['host_os']
 require 'devkit' if host_os == 'mingw32'
 
-task :default => [:test]
+task default: [:test]
 
 # Gem Spec
 gem_spec = Gem::Specification.load('commonmarker.gemspec')
@@ -33,14 +35,18 @@ end
 task 'test:unit' => :compile
 
 desc 'Run unit and conformance tests'
-task :test => %w(test:unit)
+task test: %w[test:unit]
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new(:rubocop)
 
 desc 'Run benchmarks'
 task :benchmark do
   if ENV['FETCH_PROGIT']
     `rm -rf test/progit`
     `git clone https://github.com/progit/progit.git test/progit`
-    langs = %w(ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw)
+    langs = %w[ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw]
     langs.each do |lang|
       `cat test/progit/#{lang}/*/*.markdown >> test/benchinput.md`
     end
@@ -70,13 +76,26 @@ RDoc::Task.new do |rd|
   rd.options << '--fileboxes'
 end
 
+desc 'Generate the documentation and run a web server'
+task serve: [:rdoc] do
+  require 'webrick'
+
+  puts 'Navigate to http://localhost:3000 to see the docs'
+
+  server = WEBrick::HTTPServer.new Port: 3000
+  server.mount '/', WEBrick::HTTPServlet::FileHandler, 'docs'
+  trap('INT') { server.stop }
+  server.start
+end
+
 desc 'Generate and publish docs to gh-pages'
-task :publish => [:rdoc] do
+task publish: [:rdoc] do
+  require 'tmpdir'
   require 'shellwords'
 
   Dir.mktmpdir do |tmp|
     system "mv docs/* #{tmp}"
-    system 'git checkout gh-pages'
+    system 'git checkout origin/gh-pages'
     system 'rm -rf *'
     system "mv #{tmp}/* ."
     message = Shellwords.escape("Site updated at #{Time.now.utc}")

data/bin/commonmarker

@@ -1,58 +1,118 @@
 #!/usr/bin/env ruby
-# Usage: commonmarker [--html-renderer] [--list-extensions] [--extension=EXTENSION] [FILE..]
-# Convert one or more CommonMark files to HTML and write to standard output.
-# If no FILE argument is provided, text will be read from STDIN.
-# With --html-renderer, use the HtmlRenderer renderer rather than the native C
-# renderer.
-# With --extension=EXTENSION, EXTENSION will be used for parsing, and HTML
-# output unless --html-renderer is specified.
-if ARGV.include?('--help') or ARGV.include?('-h')
-  File.read(__FILE__).split("\n").grep(/^# /).each do |line|
-    puts line[2..-1]
-  end
-  exit 0
-end
+# frozen_string_literal: true
 
-$LOAD_PATH.unshift File.join(File.dirname(__FILE__), *%w( .. lib ))
+require 'optparse'
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
 require 'commonmarker'
 
-if ARGV.include?('--version') or ARGV.include?('-v')
-  puts "commonmarker #{CommonMarker::VERSION}"
-  exit 0
-end
+root = File.expand_path('..', __dir__)
+$LOAD_PATH.unshift File.expand_path('lib', root)
 
-root = File.expand_path('../../', __FILE__)
-$:.unshift File.expand_path('lib', root)
-
-extensions = CommonMarker.extensions
-active_extensions = []
-
-renderer = nil
-ARGV.delete_if do |arg|
-  if arg =~ /^--html-renderer$/
-    renderer = true
-    true
-  elsif arg =~ /^--list-extensions$/
-    puts extensions
-    exit 0
-  elsif arg =~ /^--extension=(.+)$/
-    if extensions.include?($1)
-      active_extensions << $1.intern
-    else
-      STDERR.puts "extension #$1 not found"
-      exit 1
+def parse_options
+  options = Struct.new(:active_extensions, :active_parse_options, :active_render_options, :output_format, :renderer)
+                  .new([], [:DEFAULT], [:DEFAULT], :html)
+  extensions = CommonMarker.extensions
+  parse_options = CommonMarker::Config::OPTS.fetch(:parse)
+  render_options = CommonMarker::Config::OPTS.fetch(:render)
+  format_options = CommonMarker::Config::OPTS.fetch(:format)
+
+  option_parser = OptionParser.new do |opts|
+    opts.banner = 'Usage: commonmarker [--html-renderer] [--extension=EXTENSION]'
+    opts.separator '                    [--to=FORMAT]'
+    opts.separator '                    [--parse-option=OPTION]'
+    opts.separator '                    [--render-option=OPTION]'
+    opts.separator '                    [FILE..]'
+    opts.separator ''
+    opts.separator 'Convert one or more CommonMark files to HTML and write to standard output.'
+    opts.separator 'If no FILE argument is provided, text will be read from STDIN.'
+    opts.separator ''
+
+    opts.on('--extension=EXTENSION', Array, 'Use EXTENSION for parsing and HTML output (unless --html-renderer is specified)') do |values|
+      values.each do |value|
+        if extensions.include?(value)
+          options.active_extensions << value.to_sym
+        else
+          abort("extension '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('-h', '--help', 'Prints this help') do
+      puts opts
+      puts
+      puts "Available formats: #{format_options.join(', ')}"
+      puts "Available extentions: #{extensions.join(', ')}"
+      puts "Available parse options: #{parse_options.keys.join(', ')}"
+      puts "Available render options: #{render_options.keys.join(', ')}"
+      puts
+      puts 'See the README for more information on these.'
+      exit
+    end
+
+    opts.on('-tFORMAT', '--to=FORMAT', String, 'Specify output FORMAT') do |value|
+      value = value.to_sym
+      if format_options.include?(value)
+        options.output_format = value
+      else
+        abort("format '#{value}' not found")
+      end
+    end
+
+    opts.on('--html-renderer', 'Use the HtmlRenderer renderer rather than the native C renderer (only valid when format is html)') do
+      options.renderer = true
+    end
+
+    opts.on('--parse-option=OPTION', Array, 'OPTION passed during parsing') do |values|
+      values.each do |value|
+        if parse_options.key?(value.to_sym)
+          options.active_parse_options << value.to_sym
+        else
+          abort("parse-option '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('--render-option=OPTION', Array, 'OPTION passed during rendering') do |values|
+      values.each do |value|
+        if render_options.key?(value.to_sym)
+          options.active_render_options << value.to_sym
+        else
+          abort("render-option '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('-v', '--version', 'Version information') do
+      puts "commonmarker #{CommonMarker::VERSION}"
+      exit
     end
-    true
-  else
-    false
   end
+
+  option_parser.parse!
+
+  options
 end
 
-doc = CommonMarker.render_doc(ARGF.read, :DEFAULT, active_extensions)
+options = parse_options
+
+abort("format '#{options.output_format}' does not support using the HtmlRenderer renderer") if
+  options.renderer && options.output_format != :html
 
-if renderer
-  renderer = CommonMarker::HtmlRenderer.new(extensions: active_extensions)
-  STDOUT.write(renderer.render(doc))
-else
-  STDOUT.write(doc.to_html(:DEFAULT, active_extensions))
+doc = CommonMarker.render_doc(ARGF.read, options.active_parse_options, options.active_extensions)
+
+case options.output_format
+when :html
+  if options.renderer
+    renderer = CommonMarker::HtmlRenderer.new(options: options.active_render_options, extensions: options.active_extensions)
+    $stdout.write(renderer.render(doc))
+  else
+    $stdout.write(doc.to_html(options.active_render_options, options.active_extensions))
+  end
+when :xml
+  $stdout.write(doc.to_xml(options.active_render_options))
+when :commonmark
+  $stdout.write(doc.to_commonmark(options.active_render_options))
+when :plaintext
+  $stdout.write(doc.to_plaintext(options.active_render_options))
 end

data/commonmarker.gemspec

@@ -1,5 +1,6 @@
-# encoding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'commonmarker/version'
 
@@ -8,29 +9,31 @@ Gem::Specification.new do |s|
   s.version = CommonMarker::VERSION
   s.summary = 'CommonMark parser and renderer. Written in C, wrapped in Ruby.'
   s.description = 'A fast, safe, extensible parser for CommonMark. This wraps the official libcmark library.'
-  s.authors       = ['Garen Torikian', 'Ashe Connor']
-  s.homepage = 'http://github.com/gjtorikian/commonmarker'
+  s.authors = ['Garen Torikian', 'Ashe Connor']
+  s.homepage = 'https://github.com/gjtorikian/commonmarker'
   s.license = 'MIT'
-  s.required_ruby_version = '>= 2.0.0'
 
-  s.files         = %w(LICENSE.txt README.md Rakefile commonmarker.gemspec bin/commonmarker)
+  s.files         = %w[LICENSE.txt README.md Rakefile commonmarker.gemspec bin/commonmarker]
   s.files        += Dir.glob('lib/**/*.rb')
   s.files        += Dir.glob('ext/commonmarker/*.*')
-  s.test_files    = Dir.glob('test/**/*')
+  s.test_files    = Dir.glob('test/**/*').reject { |f| f == 'test/benchinput.md' || f.start_with?('test/progit/') }
   s.extensions    = ['ext/commonmarker/extconf.rb']
 
-  s.test_files = s.files.grep(%r{^test/})
   s.executables = ['commonmarker']
-  s.require_paths = %w(lib ext)
+  s.require_paths = %w[lib ext]
+  s.required_ruby_version = ['>= 2.6', '< 4.0']
 
-  s.rdoc_options += ['-x', 'ext/commonmarker/cmark/.*']
+  s.metadata['rubygems_mfa_required'] = 'true'
 
-  s.add_dependency 'ruby-enum', '~> 0.5'
+  s.rdoc_options += ['-x', 'ext/commonmarker/cmark/.*']
 
+  s.add_development_dependency 'awesome_print'
+  s.add_development_dependency 'json', '~> 2.3'
   s.add_development_dependency 'minitest', '~> 5.6'
+  s.add_development_dependency 'minitest-focus', '~> 1.1'
+  s.add_development_dependency 'rake'
   s.add_development_dependency 'rake-compiler', '~> 0.9'
-  s.add_development_dependency 'bundler', '~> 1.2'
-  s.add_development_dependency 'json', '~> 1.8.1'
-  s.add_development_dependency 'awesome_print'
-  s.add_development_dependency 'rdoc', '~> 5.1'
+  s.add_development_dependency 'rdoc', '~> 6.2'
+  s.add_development_dependency 'rubocop'
+  s.add_development_dependency 'rubocop-standard'
 end

data/ext/commonmarker/autolink.c

@@ -269,7 +269,11 @@ static cmark_node *match(cmark_syntax_extension *ext, cmark_parser *parser,
   // inline was finished in inlines.c.
 }
 
-static void postprocess_text(cmark_parser *parser, cmark_node *text, int offset) {
+static void postprocess_text(cmark_parser *parser, cmark_node *text, int offset, int depth) {
+  // postprocess_text can recurse very deeply if there is a very long line of
+  // '@' only.  Stop at a reasonable depth to ensure it cannot crash.
+  if (depth > 1000) return;
+
   size_t link_end;
   uint8_t *data = text->as.literal.data,
     *at;
@@ -307,7 +311,7 @@ static void postprocess_text(cmark_parser *parser, cmark_node *text, int offset)
   }
 
   if (rewind == 0 || ns > 0) {
-    postprocess_text(parser, text, max_rewind + 1 + offset);
+    postprocess_text(parser, text, max_rewind + 1 + offset, depth + 1);
     return;
   }
 
@@ -327,14 +331,14 @@ static void postprocess_text(cmark_parser *parser, cmark_node *text, int offset)
 
   if (link_end < 2 || nb != 1 || np == 0 ||
       (!cmark_isalpha(data[link_end - 1]) && data[link_end - 1] != '.')) {
-    postprocess_text(parser, text, max_rewind + 1 + offset);
+    postprocess_text(parser, text, max_rewind + 1 + offset, depth + 1);
     return;
   }
 
   link_end = autolink_delim(data, link_end);
 
   if (link_end == 0) {
-    postprocess_text(parser, text, max_rewind + 1 + offset);
+    postprocess_text(parser, text, max_rewind + 1 + offset, depth + 1);
     return;
   }
 
@@ -369,7 +373,7 @@ static void postprocess_text(cmark_parser *parser, cmark_node *text, int offset)
   text->as.literal.len = offset + max_rewind - rewind;
   text->as.literal.data[text->as.literal.len] = 0;
 
-  postprocess_text(parser, post, 0);
+  postprocess_text(parser, post, 0, depth + 1);
 }
 
 static cmark_node *postprocess(cmark_syntax_extension *ext, cmark_parser *parser, cmark_node *root) {
@@ -396,7 +400,7 @@ static cmark_node *postprocess(cmark_syntax_extension *ext, cmark_parser *parser
     }
 
     if (ev == CMARK_EVENT_ENTER && node->type == CMARK_NODE_TEXT) {
-      postprocess_text(parser, node, 0);
+      postprocess_text(parser, node, 0, /*depth*/0);
     }
   }