commonmarker 0.19.0 → 0.21.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a9c068146510f80180b4524934cf90ba56044100
-  data.tar.gz: 5d5ff7d6b1a68340d8e330d6b6a1e9b1d01df944
+SHA256:
+  metadata.gz: 1a1380e99f120ae24a27bae6244ffeab5b0b3fe828155828cf8c092d8c83471a
+  data.tar.gz: 92c85125c0e4d6f8673312e64e67feda6993a4fc1250661fad3714836bc96454
 SHA512:
-  metadata.gz: 828329dac75b03c52a50674a53a994791bc6c26fb05fc7fed4377ff241e7b87560b3dd9b2aa8be0b0a4e2a7ac194500a4e584872c6a1bc44618a13e3cad741f8
-  data.tar.gz: 5bf814489204bf76b497cd8dd570dd88b1f8af8f9a45847dad091b756b243ae618466e8e85441c4e39a0d22191e3a2001ee15c8309a80ec93f630952c92e0108
+  metadata.gz: 5f136c3413348cea8e3a0f3e8b0242ffc8bba92b9004278170ee7d2532e87b6c5d6af73c952d2c31d509143ae2cfe63d96aeca6ff4749a599fefbcad7580214e
+  data.tar.gz: 4d33d9cb161d0b484ed9bc353c9c01dd3f8715f533c05a2f7f61a16792433153d19324d1b62a0ae6b6750441917843495349a1140e93d2b1960794ea0346e444

data/README.md

@@ -112,10 +112,8 @@ class MyHtmlRenderer < CommonMarker::HtmlRenderer
   end
 end
 
-# this renderer prints directly to STDOUT, instead
-# of returning a string
 myrenderer = MyHtmlRenderer.new
-print(myrenderer.render(doc))
+puts myrenderer.render(doc)
 
 # Print any warnings to STDERR
 renderer.warnings.each do |w|
@@ -176,6 +174,7 @@ Both `render_html` and `render_doc` take an optional third argument defining the
 The available extensions are:
 
 * `:table` - This provides support for tables.
+* `:tasklist` - This provides support for task list items.
 * `:strikethrough` - This provides support for strikethroughs.
 * `:autolink` - This provides support for automatically converting URLs to anchor tags.
 * `:tagfilter` - This escapes [several "unsafe" HTML tags](https://github.github.com/gfm/#disallowed-raw-html-extension-), causing them to not have any effect.

data/Rakefile

@@ -5,7 +5,6 @@ require 'rake/clean'
 require 'rake/extensiontask'
 require 'digest/md5'
 
-
 host_os = RbConfig::CONFIG['host_os']
 require 'devkit' if host_os == 'mingw32'
 
@@ -36,7 +35,7 @@ end
 task 'test:unit' => :compile
 
 desc 'Run unit and conformance tests'
-task test: %w(test:unit)
+task test: %w[test:unit]
 
 require 'rubocop/rake_task'
 
@@ -47,7 +46,7 @@ task :benchmark do
   if ENV['FETCH_PROGIT']
     `rm -rf test/progit`
     `git clone https://github.com/progit/progit.git test/progit`
-    langs = %w(ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw)
+    langs = %w[ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw]
     langs.each do |lang|
       `cat test/progit/#{lang}/*/*.markdown >> test/benchinput.md`
     end
@@ -77,13 +76,26 @@ RDoc::Task.new do |rd|
   rd.options << '--fileboxes'
 end
 
+desc 'Generate the documentation and run a web server'
+task serve: [:rdoc] do
+  require 'webrick'
+
+  puts 'Navigate to http://localhost:3000 to see the docs'
+
+  server = WEBrick::HTTPServer.new Port: 3000
+  server.mount '/', WEBrick::HTTPServlet::FileHandler, 'docs'
+  trap('INT') { server.stop }
+  server.start
+end
+
 desc 'Generate and publish docs to gh-pages'
 task publish: [:rdoc] do
+  require 'tmpdir'
   require 'shellwords'
 
   Dir.mktmpdir do |tmp|
     system "mv docs/* #{tmp}"
-    system 'git checkout gh-pages'
+    system 'git checkout origin/gh-pages'
     system 'rm -rf *'
     system "mv #{tmp}/* ."
     message = Shellwords.escape("Site updated at #{Time.now.utc}")

data/bin/commonmarker

@@ -1,60 +1,98 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Usage: commonmarker [--html-renderer] [--list-extensions] [--extension=EXTENSION] [FILE..]
-# Convert one or more CommonMark files to HTML and write to standard output.
-# If no FILE argument is provided, text will be read from STDIN.
-# With --html-renderer, use the HtmlRenderer renderer rather than the native C
-# renderer.
-# With --extension=EXTENSION, EXTENSION will be used for parsing, and HTML
-# output unless --html-renderer is specified.
-if ARGV.include?('--help') or ARGV.include?('-h')
-  File.read(__FILE__).split("\n").grep(/^# /).each do |line|
-    puts line[2..-1]
-  end
-  exit 0
-end
+require 'optparse'
+require 'ostruct'
 
 $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
 require 'commonmarker'
 
-if ARGV.include?('--version') or ARGV.include?('-v')
-  puts "commonmarker #{CommonMarker::VERSION}"
-  exit 0
-end
+root = File.expand_path('..', __dir__)
+$LOAD_PATH.unshift File.expand_path('lib', root)
+
+def parse_options
+  options = OpenStruct.new
+  extensions = CommonMarker.extensions
+  parse_options = CommonMarker::Config::Parse
+  render_options = CommonMarker::Config::Render
+
+  options.active_extensions = []
+  options.active_parse_options = [:DEFAULT]
+  options.active_render_options = [:DEFAULT]
+
+  option_parser = OptionParser.new do |opts|
+    opts.banner = 'Usage: commonmarker [--html-renderer] [--extension=EXTENSION]'
+    opts.separator '                    [--parse-option=OPTION]'
+    opts.separator '                    [--render-option=OPTION]'
+    opts.separator '                    [FILE..]'
+    opts.separator ''
+    opts.separator 'Convert one or more CommonMark files to HTML and write to standard output.'
+    opts.separator 'If no FILE argument is provided, text will be read from STDIN.'
+    opts.separator ''
+
+    opts.on('--extension=EXTENSION', Array, 'Use EXTENSION for parsing and HTML output (unless --html-renderer is specified)') do |values|
+      values.each do |value|
+        if extensions.include?(value)
+          options.active_extensions << value.to_sym
+        else
+          abort("extension '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('-h', '--help', 'Prints this help') do
+      puts opts
+      puts
+      puts "Available extentions: #{extensions.join(', ')}"
+      puts "Available parse options: #{parse_options.keys.join(', ')}"
+      puts "Available render options: #{render_options.keys.join(', ')}"
+      puts
+      puts 'See the README for more information on these.'
+      exit
+    end
 
-root = File.expand_path('../../', __FILE__)
-$:.unshift File.expand_path('lib', root)
-
-extensions = CommonMarker.extensions
-active_extensions = []
-
-renderer = nil
-ARGV.delete_if do |arg|
-  if arg =~ /^--html-renderer$/
-    renderer = true
-    true
-  elsif arg =~ /^--list-extensions$/
-    puts extensions
-    exit 0
-  elsif arg =~ /^--extension=(.+)$/
-    if extensions.include?($1)
-      active_extensions << $1.intern
-    else
-      STDERR.puts "extension #$1 not found"
-      exit 1
+    opts.on('--html-renderer', 'Use the HtmlRenderer renderer rather than the native C renderer') do
+      options.renderer = true
+    end
+
+    opts.on('--parse-option=OPTION', Array, 'OPTION passed during parsing') do |values|
+      values.each do |value|
+        if parse_options.key?(value.to_sym)
+          options.active_parse_options << value.to_sym
+        else
+          abort("parse-option '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('--render-option=OPTION', Array, 'OPTION passed during rendering') do |values|
+      values.each do |value|
+        if render_options.key?(value.to_sym)
+          options.active_render_options << value.to_sym
+        else
+          abort("render-option '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('-v', '--version', 'Version information') do
+      puts "commonmarker #{CommonMarker::VERSION}"
+      exit
     end
-    true
-  else
-    false
   end
+
+  option_parser.parse!
+
+  options
 end
 
-doc = CommonMarker.render_doc(ARGF.read, :DEFAULT, active_extensions)
+options = parse_options
+
+doc = CommonMarker.render_doc(ARGF.read, options.active_parse_options, options.active_extensions)
 
-if renderer
-  renderer = CommonMarker::HtmlRenderer.new(extensions: active_extensions)
+if options.renderer
+  renderer = CommonMarker::HtmlRenderer.new(extensions: options.active_extensions)
   STDOUT.write(renderer.render(doc))
 else
-  STDOUT.write(doc.to_html(:DEFAULT, active_extensions))
+  STDOUT.write(doc.to_html(options.active_render_options, options.active_extensions))
 end

data/commonmarker.gemspec

@@ -1,7 +1,6 @@
-# encoding: utf-8
 # frozen_string_literal: true
 
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'commonmarker/version'
 
@@ -10,32 +9,30 @@ Gem::Specification.new do |s|
   s.version = CommonMarker::VERSION
   s.summary = 'CommonMark parser and renderer. Written in C, wrapped in Ruby.'
   s.description = 'A fast, safe, extensible parser for CommonMark. This wraps the official libcmark library.'
-  s.authors       = ['Garen Torikian', 'Ashe Connor']
+  s.authors = ['Garen Torikian', 'Ashe Connor']
   s.homepage = 'https://github.com/gjtorikian/commonmarker'
   s.license = 'MIT'
-  s.required_ruby_version = '>= 2.0.0'
 
-  s.files         = %w(LICENSE.txt README.md Rakefile commonmarker.gemspec bin/commonmarker)
+  s.files         = %w[LICENSE.txt README.md Rakefile commonmarker.gemspec bin/commonmarker]
   s.files        += Dir.glob('lib/**/*.rb')
   s.files        += Dir.glob('ext/commonmarker/*.*')
-  s.test_files    = Dir.glob('test/**/*')
+  s.test_files    = Dir.glob('test/**/*').reject { |f| f == 'test/benchinput.md' || f.start_with?('test/progit/') }
   s.extensions    = ['ext/commonmarker/extconf.rb']
 
-  s.test_files = s.files.grep(%r{^test/})
   s.executables = ['commonmarker']
-  s.require_paths = %w(lib ext)
+  s.require_paths = %w[lib ext]
 
   s.rdoc_options += ['-x', 'ext/commonmarker/cmark/.*']
 
   s.add_dependency 'ruby-enum', '~> 0.5'
 
+  s.add_development_dependency 'awesome_print'
+  s.add_development_dependency 'json', '~> 2.3'
   s.add_development_dependency 'minitest', '~> 5.6'
   s.add_development_dependency 'minitest-focus', '~> 1.1'
-  s.add_development_dependency 'rake-compiler', '~> 0.9'
-  s.add_development_dependency 'json', '~> 1.8.1'
-  s.add_development_dependency 'awesome_print'
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'rdoc', '~> 5.1'
+  s.add_development_dependency 'rake-compiler', '~> 0.9'
+  s.add_development_dependency 'rdoc', '~> 6.2'
   s.add_development_dependency 'rubocop'
   s.add_development_dependency 'rubocop-standard'
 end

data/ext/commonmarker/blocks.c

@@ -36,6 +36,10 @@ static bool S_last_line_blank(const cmark_node *node) {
   return (node->flags & CMARK_NODE__LAST_LINE_BLANK) != 0;
 }
 
+static bool S_last_line_checked(const cmark_node *node) {
+  return (node->flags & CMARK_NODE__LAST_LINE_CHECKED) != 0;
+}
+
 static CMARK_INLINE cmark_node_type S_type(const cmark_node *node) {
   return (cmark_node_type)node->type;
 }
@@ -47,6 +51,10 @@ static void S_set_last_line_blank(cmark_node *node, bool is_blank) {
     node->flags &= ~CMARK_NODE__LAST_LINE_BLANK;
 }
 
+static void S_set_last_line_checked(cmark_node *node) {
+  node->flags |= CMARK_NODE__LAST_LINE_CHECKED;
+}
+
 static CMARK_INLINE bool S_is_line_end_char(char c) {
   return (c == '\n' || c == '\r');
 }
@@ -121,8 +129,6 @@ static void cmark_parser_reset(cmark_parser *parser) {
   parser->root = document;
   parser->current = document;
 
-  parser->last_buffer_ended_with_cr = false;
-
   parser->syntax_extensions = saved_exts;
   parser->inline_syntax_extensions = saved_inline_exts;
   parser->options = saved_options;
@@ -234,19 +240,35 @@ static void remove_trailing_blank_lines(cmark_strbuf *ln) {
 
 // Check to see if a node ends with a blank line, descending
 // if needed into lists and sublists.
-static bool ends_with_blank_line(cmark_node *node) {
-  cmark_node *cur = node;
-  while (cur != NULL) {
-    if (S_last_line_blank(cur)) {
-      return true;
-    }
-    if (S_type(cur) == CMARK_NODE_LIST || S_type(cur) == CMARK_NODE_ITEM) {
-      cur = cur->last_child;
-    } else {
-      cur = NULL;
-    }
+static bool S_ends_with_blank_line(cmark_node *node) {
+  if (S_last_line_checked(node)) {
+    return(S_last_line_blank(node));
+  } else if ((S_type(node) == CMARK_NODE_LIST ||
+              S_type(node) == CMARK_NODE_ITEM) && node->last_child) {
+    S_set_last_line_checked(node);
+    return(S_ends_with_blank_line(node->last_child));
+  } else {
+    S_set_last_line_checked(node);
+    return (S_last_line_blank(node));
   }
-  return false;
+}
+
+// returns true if content remains after link defs are resolved.
+static bool resolve_reference_link_definitions(
+		cmark_parser *parser,
+                cmark_node *b) {
+  bufsize_t pos;
+  cmark_strbuf *node_content = &b->content;
+  cmark_chunk chunk = {node_content->ptr, node_content->size, 0};
+  while (chunk.len && chunk.data[0] == '[' &&
+         (pos = cmark_parse_reference_inline(parser->mem, &chunk,
+					     parser->refmap))) {
+
+    chunk.data += pos;
+    chunk.len -= pos;
+  }
+  cmark_strbuf_drop(node_content, (node_content->size - chunk.len));
+  return !is_blank(&b->content, 0);
 }
 
 static cmark_node *finalize(cmark_parser *parser, cmark_node *b) {
@@ -254,6 +276,7 @@ static cmark_node *finalize(cmark_parser *parser, cmark_node *b) {
   cmark_node *item;
   cmark_node *subitem;
   cmark_node *parent;
+  bool has_content;
 
   parent = b->parent;
   assert(b->flags &
@@ -283,15 +306,8 @@ static cmark_node *finalize(cmark_parser *parser, cmark_node *b) {
   switch (S_type(b)) {
   case CMARK_NODE_PARAGRAPH:
   {
-    cmark_chunk chunk = {node_content->ptr, node_content->size, 0};
-    while (chunk.len && chunk.data[0] == '[' &&
-           (pos = cmark_parse_reference_inline(parser->mem, &chunk, parser->refmap))) {
-
-      chunk.data += pos;
-      chunk.len -= pos;
-    }
-    cmark_strbuf_drop(node_content, (node_content->size - chunk.len));
-    if (is_blank(node_content, 0)) {
+    has_content = resolve_reference_link_definitions(parser, b);
+    if (!has_content) {
       // remove blank node (former reference def)
       cmark_node_free(b);
     }
@@ -343,7 +359,8 @@ static cmark_node *finalize(cmark_parser *parser, cmark_node *b) {
       // spaces between them:
       subitem = item->first_child;
       while (subitem) {
-        if (ends_with_blank_line(subitem) && (item->next || subitem->next)) {
+        if ((item->next || subitem->next) &&
+            S_ends_with_blank_line(subitem)) {
           b->as.list.tight = false;
           break;
         }
@@ -748,6 +765,40 @@ static void chop_trailing_hashtags(cmark_chunk *ch) {
   }
 }
 
+// Check for thematic break.  On failure, return 0 and update
+// thematic_break_kill_pos with the index at which the
+// parse fails.  On success, return length of match.
+// "...three or more hyphens, asterisks,
+// or underscores on a line by themselves. If you wish, you may use
+// spaces between the hyphens or asterisks."
+static int S_scan_thematic_break(cmark_parser *parser, cmark_chunk *input,
+                                 bufsize_t offset) {
+  bufsize_t i;
+  char c;
+  char nextc = '\0';
+  int count;
+  i = offset;
+  c = peek_at(input, i);
+  if (!(c == '*' || c == '_' || c == '-')) {
+    parser->thematic_break_kill_pos = i;
+    return 0;
+  }
+  count = 1;
+  while ((nextc = peek_at(input, ++i))) {
+    if (nextc == c) {
+      count++;
+    } else if (nextc != ' ' && nextc != '\t') {
+      break;
+    }
+  }
+  if (count >= 3 && (nextc == '\r' || nextc == '\n')) {
+    return (i - offset) + 1;
+  } else {
+    parser->thematic_break_kill_pos = i;
+    return 0;
+  }
+}
+
 // Find first nonspace character from current offset, setting
 // parser->first_nonspace, parser->first_nonspace_column,
 // parser->indent, and parser->blank. Does not advance parser->offset.
@@ -1040,6 +1091,7 @@ static void open_new_blocks(cmark_parser *parser, cmark_node **container,
   bufsize_t matched = 0;
   int lev = 0;
   bool save_partially_consumed_tab;
+  bool has_content;
   int save_offset;
   int save_column;
 
@@ -1112,13 +1164,20 @@ static void open_new_blocks(cmark_parser *parser, cmark_node **container,
     } else if (!indented && cont_type == CMARK_NODE_PARAGRAPH &&
                (lev =
                     scan_setext_heading_line(input, parser->first_nonspace))) {
-      (*container)->type = (uint16_t)CMARK_NODE_HEADING;
-      (*container)->as.heading.level = lev;
-      (*container)->as.heading.setext = true;
-      S_advance_offset(parser, input, input->len - 1 - parser->offset, false);
+      // finalize paragraph, resolving reference links
+      has_content = resolve_reference_link_definitions(parser, *container);
+
+      if (has_content) {
+
+        (*container)->type = (uint16_t)CMARK_NODE_HEADING;
+        (*container)->as.heading.level = lev;
+        (*container)->as.heading.setext = true;
+        S_advance_offset(parser, input, input->len - 1 - parser->offset, false);
+      }
     } else if (!indented &&
                !(cont_type == CMARK_NODE_PARAGRAPH && !all_matched) &&
-               (matched = scan_thematic_break(input, parser->first_nonspace))) {
+	       (parser->thematic_break_kill_pos <= parser->first_nonspace) &&
+               (matched = S_scan_thematic_break(parser, input, parser->first_nonspace))) {
       // it's only now that we know the line is not part of a setext heading:
       *container = add_child(parser, *container, CMARK_NODE_THEMATIC_BREAK,
                              parser->first_nonspace + 1);
@@ -1377,6 +1436,7 @@ static void S_process_line(cmark_parser *parser, const unsigned char *buffer,
   parser->column = 0;
   parser->first_nonspace = 0;
   parser->first_nonspace_column = 0;
+  parser->thematic_break_kill_pos = 0;
   parser->indent = 0;
   parser->blank = false;
   parser->partially_consumed_tab = false;