commonmarker 0.21.2 → 0.23.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac3365deb1dd8288503331626fc20ec583a5f96a23e25d330f4ff47e526bfc54
-  data.tar.gz: 975bcb958eb2e994a15090311088ff95870af95b1fc2d73bc1c4eb216fe69388
+  metadata.gz: bc6d71b6e85fb61abc438252b07369d3264982cf8fb327d96e5e8333bb23841e
+  data.tar.gz: 1911965561ddf20104db09c44c4e5d25f65a277a28b825bc0c03bb172779a623
 SHA512:
-  metadata.gz: 531cda0280a75a40c139969287106646c8b2cef181b48249055f3a896bc4329485a0f70ac59f6f3a85eed00247c15f4a6785f478b957ebd3594be3074a9d25a9
-  data.tar.gz: 3bb88a326ea1e0e7f99da2b459b90cac2ea748e57d933029c11278f230064a433a00889c93d65851dfef4b131ba46562a33d4597ef677f3e382325a712b0ffe6
+  metadata.gz: 59737cb433c3f4f3d53e06d3466f1f3dcd8a40e201e8b0f26be4d553f45667137e05d980100304e720d0e8fda178da3eaa9e421608cf4e53c6d159ff9261ae3e
+  data.tar.gz: c3088c3219da29bdb0356f46a5b7a7f2881410c825e101f7ced3964760bfcfcb5cef57e7f3ebf5bd9f686b974ad80887fc93899e616afb211dc0e648d6352cd2

data/README.md

@@ -1,6 +1,6 @@
 # CommonMarker
 
-[![Build Status](https://travis-ci.org/gjtorikian/commonmarker.svg)](https://travis-ci.org/gjtorikian/commonmarker) [![Gem Version](https://badge.fury.io/rb/commonmarker.svg)](http://badge.fury.io/rb/commonmarker)
+![Build Status](https://github.com/gjtorikian/commonmarker/workflows/CI/badge.svg) [![Gem Version](https://badge.fury.io/rb/commonmarker.svg)](http://badge.fury.io/rb/commonmarker)
 
 Ruby wrapper for [libcmark-gfm](https://github.com/github/cmark),
 GitHub's fork of the reference parser for CommonMark. It passes all of the C tests, and is therefore spec-complete. It also includes extensions to the CommonMark spec as documented in the [GitHub Flavored Markdown spec](http://github.github.com/gfm/), such as support for tables, strikethroughs, and autolinking.
@@ -130,26 +130,31 @@ CommonMarker accepts the same options that CMark does, as symbols. Note that the
 | Name                          | Description
 | ----------------------------- | -----------
 | `:DEFAULT`                    | The default parsing system.
+| `:SOURCEPOS`                  | Include source position in nodes
 | `:UNSAFE`                     | Allow raw/custom HTML and unsafe links.
-| `:FOOTNOTES`                  | Parse footnotes.
-| `:LIBERAL_HTML_TAG`           | Support liberal parsing of inline HTML tags.
+| `:VALIDATE_UTF8`              | Replace illegal sequences with the replacement character `U+FFFD`.
 | `:SMART`                      | Use smart punctuation (curly quotes, etc.).
+| `:LIBERAL_HTML_TAG`           | Support liberal parsing of inline HTML tags.
+| `:FOOTNOTES`                  | Parse footnotes.
 | `:STRIKETHROUGH_DOUBLE_TILDE` | Parse strikethroughs by double tildes (compatibility with [redcarpet](https://github.com/vmg/redcarpet))
-| `:VALIDATE_UTF8`              | Replace illegal sequences with the replacement character `U+FFFD`.
 
 ### Render options
 
 | Name                             | Description                                                     |
 | ------------------               | -----------                                                     |
 | `:DEFAULT`                       | The default rendering system.                                   |
-| `:UNSAFE`                        | Allow raw/custom HTML and unsafe links.                         |
-| `:GITHUB_PRE_LANG`               | Use GitHub-style `<pre lang>` for fenced code blocks.           |
+| `:SOURCEPOS`                     | Include source position in rendered HTML.                       |
 | `:HARDBREAKS`                    | Treat `\n` as hardbreaks (by adding `<br/>`).                   |
+| `:UNSAFE`                        | Allow raw/custom HTML and unsafe links.                         |
 | `:NOBREAKS`                      | Translate `\n` in the source to a single whitespace.            |
-| `:SOURCEPOS`                     | Include source position in rendered HTML.                       |
+| `:VALIDATE_UTF8`                 | Replace illegal sequences with the replacement character `U+FFFD`. |
+| `:SMART`                         | Use smart punctuation (curly quotes, etc.).                     |
+| `:GITHUB_PRE_LANG`               | Use GitHub-style `<pre lang>` for fenced code blocks.           |
+| `:LIBERAL_HTML_TAG`              | Support liberal parsing of inline HTML tags.                    |
+| `:FOOTNOTES`                     | Render footnotes.                                               |
+| `:STRIKETHROUGH_DOUBLE_TILDE`    | Parse strikethroughs by double tildes (compatibility with [redcarpet](https://github.com/vmg/redcarpet)) |
 | `:TABLE_PREFER_STYLE_ATTRIBUTES` | Use `style` insted of `align` for table cells.                  |
 | `:FULL_INFO_STRING`              | Include full info strings of code blocks in separate attribute. |
-| `:FOOTNOTES`                     | Render footnotes.                                               |
 
 ### Passing options
 
@@ -180,6 +185,76 @@ The available extensions are:
 * `:autolink` - This provides support for automatically converting URLs to anchor tags.
 * `:tagfilter` - This escapes [several "unsafe" HTML tags](https://github.github.com/gfm/#disallowed-raw-html-extension-), causing them to not have any effect.
 
+## Output formats
+
+Like CMark, CommonMarker can generate output in several formats: HTML, XML, plaintext, and commonmark are currently supported.
+
+### HTML
+
+The default output format, HTML, will be generated when calling `to_html` or using `--to=html` on the command line.
+
+```ruby
+doc = CommonMarker.render_doc('*Hello* world!', :DEFAULT)
+puts(doc.to_html)
+
+<p><em>Hello</em> world!</p>
+```
+
+### XML
+
+XML will be generated when calling `to_xml` or using `--to=xml` on the command line.
+
+```ruby
+doc = CommonMarker.render_doc('*Hello* world!', :DEFAULT)
+puts(doc.to_xml)
+
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE document SYSTEM "CommonMark.dtd">
+<document xmlns="http://commonmark.org/xml/1.0">
+  <paragraph>
+    <emph>
+      <text xml:space="preserve">Hello</text>
+    </emph>
+    <text xml:space="preserve"> world!</text>
+  </paragraph>
+</document>
+```
+
+### Plaintext
+
+Plaintext will be generated when calling `to_plaintext` or using `--to=plaintext` on the command line.
+
+```ruby
+doc = CommonMarker.render_doc('*Hello* world!', :DEFAULT)
+puts(doc.to_plaintext)
+
+Hello world!
+```
+
+### Commonmark
+
+Commonmark will be generated when calling `to_commonmark` or using `--to=commonmark` on the command line.
+
+``` ruby
+text = <<-TEXT
+1. I am a numeric list.
+2. I continue the list.
+* Suddenly, an unordered list!
+* What fun!
+TEXT
+
+doc = CommonMarker.render_doc(text, :DEFAULT)
+puts(doc.to_commonmark)
+
+1.  I am a numeric list.
+2.  I continue the list.
+
+<!-- end list -->
+
+  - Suddenly, an unordered list\!
+  - What fun\!
+```
+
 ## Developing locally
 
 After cloning the repo:

data/bin/commonmarker

@@ -2,7 +2,6 @@
 # frozen_string_literal: true
 
 require 'optparse'
-require 'ostruct'
 
 $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
 require 'commonmarker'
@@ -11,17 +10,16 @@ root = File.expand_path('..', __dir__)
 $LOAD_PATH.unshift File.expand_path('lib', root)
 
 def parse_options
-  options = OpenStruct.new
+  options = Struct.new(:active_extensions, :active_parse_options, :active_render_options, :output_format, :renderer)
+                  .new([], [:DEFAULT], [:DEFAULT], :html)
   extensions = CommonMarker.extensions
-  parse_options = CommonMarker::Config::Parse
-  render_options = CommonMarker::Config::Render
-
-  options.active_extensions = []
-  options.active_parse_options = [:DEFAULT]
-  options.active_render_options = [:DEFAULT]
+  parse_options = CommonMarker::Config::OPTS.fetch(:parse)
+  render_options = CommonMarker::Config::OPTS.fetch(:render)
+  format_options = CommonMarker::Config::OPTS.fetch(:format)
 
   option_parser = OptionParser.new do |opts|
     opts.banner = 'Usage: commonmarker [--html-renderer] [--extension=EXTENSION]'
+    opts.separator '                    [--to=FORMAT]'
     opts.separator '                    [--parse-option=OPTION]'
     opts.separator '                    [--render-option=OPTION]'
     opts.separator '                    [FILE..]'
@@ -43,6 +41,7 @@ def parse_options
     opts.on('-h', '--help', 'Prints this help') do
       puts opts
       puts
+      puts "Available formats: #{format_options.join(', ')}"
       puts "Available extentions: #{extensions.join(', ')}"
       puts "Available parse options: #{parse_options.keys.join(', ')}"
       puts "Available render options: #{render_options.keys.join(', ')}"
@@ -51,7 +50,16 @@ def parse_options
       exit
     end
 
-    opts.on('--html-renderer', 'Use the HtmlRenderer renderer rather than the native C renderer') do
+    opts.on('-tFORMAT', '--to=FORMAT', String, 'Specify output FORMAT') do |value|
+      value = value.to_sym
+      if format_options.include?(value)
+        options.output_format = value
+      else
+        abort("format '#{value}' not found")
+      end
+    end
+
+    opts.on('--html-renderer', 'Use the HtmlRenderer renderer rather than the native C renderer (only valid when format is html)') do
       options.renderer = true
     end
 
@@ -88,11 +96,23 @@ end
 
 options = parse_options
 
+abort("format '#{options.output_format}' does not support using the HtmlRenderer renderer") if
+  options.renderer && options.output_format != :html
+
 doc = CommonMarker.render_doc(ARGF.read, options.active_parse_options, options.active_extensions)
 
-if options.renderer
-  renderer = CommonMarker::HtmlRenderer.new(extensions: options.active_extensions)
-  $stdout.write(renderer.render(doc))
-else
-  $stdout.write(doc.to_html(options.active_render_options, options.active_extensions))
+case options.output_format
+when :html
+  if options.renderer
+    renderer = CommonMarker::HtmlRenderer.new(options: options.active_render_options, extensions: options.active_extensions)
+    $stdout.write(renderer.render(doc))
+  else
+    $stdout.write(doc.to_html(options.active_render_options, options.active_extensions))
+  end
+when :xml
+  $stdout.write(doc.to_xml(options.active_render_options))
+when :commonmark
+  $stdout.write(doc.to_commonmark(options.active_render_options))
+when :plaintext
+  $stdout.write(doc.to_plaintext(options.active_render_options))
 end

data/commonmarker.gemspec

@@ -21,11 +21,11 @@ Gem::Specification.new do |s|
 
   s.executables = ['commonmarker']
   s.require_paths = %w[lib ext]
-  s.required_ruby_version = ['>= 2.4.10', '< 4.0']
+  s.required_ruby_version = ['>= 2.6', '< 4.0']
 
-  s.rdoc_options += ['-x', 'ext/commonmarker/cmark/.*']
+  s.metadata['rubygems_mfa_required'] = 'true'
 
-  s.add_dependency 'ruby-enum', '~> 0.5'
+  s.rdoc_options += ['-x', 'ext/commonmarker/cmark/.*']
 
   s.add_development_dependency 'awesome_print'
   s.add_development_dependency 'json', '~> 2.3'

data/ext/commonmarker/blocks.c

@@ -468,7 +468,6 @@ static void process_footnotes(cmark_parser *parser) {
   while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) {
     cur = cmark_iter_get_node(iter);
     if (ev_type == CMARK_EVENT_EXIT && cur->type == CMARK_NODE_FOOTNOTE_DEFINITION) {
-      cmark_node_unlink(cur);
       cmark_footnote_create(map, cur);
     }
   }
@@ -485,6 +484,15 @@ static void process_footnotes(cmark_parser *parser) {
         if (!footnote->ix)
           footnote->ix = ++ix;
 
+        // store a reference to this footnote reference's footnote definition
+        // this is used by renderers when generating label ids
+        cur->parent_footnote_def = footnote->node;
+
+        // keep track of a) count of how many times this footnote def has been
+        // referenced, and b) which reference index this footnote ref is at.
+        // this is used by renderers when generating links and backreferences.
+        cur->footnote.ref_ix = ++footnote->node->footnote.def_count;
+
         char n[32];
         snprintf(n, sizeof(n), "%d", footnote->ix);
         cmark_chunk_free(parser->mem, &cur->as.literal);
@@ -515,13 +523,16 @@ static void process_footnotes(cmark_parser *parser) {
     qsort(map->sorted, map->size, sizeof(cmark_map_entry *), sort_footnote_by_ix);
     for (unsigned int i = 0; i < map->size; ++i) {
       cmark_footnote *footnote = (cmark_footnote *)map->sorted[i];
-      if (!footnote->ix)
+      if (!footnote->ix) {
+        cmark_node_unlink(footnote->node);
         continue;
+      }
       cmark_node_append_child(parser->root, footnote->node);
       footnote->node = NULL;
     }
   }
 
+  cmark_unlink_footnotes_map(map);
   cmark_map_free(map);
 }
 

data/ext/commonmarker/cmark-gfm_version.h

@@ -1,7 +1,7 @@
 #ifndef CMARK_GFM_VERSION_H
 #define CMARK_GFM_VERSION_H
 
-#define CMARK_GFM_VERSION ((0 << 24) | (29 << 16) | (0 << 8) | 0)
-#define CMARK_GFM_VERSION_STRING "0.29.0.gfm.0"
+#define CMARK_GFM_VERSION ((0 << 24) | (29 << 16) | (0 << 8) | 3)
+#define CMARK_GFM_VERSION_STRING "0.29.0.gfm.3"
 
 #endif

data/ext/commonmarker/commonmark.c

@@ -477,7 +477,13 @@ static int S_render_node(cmark_renderer *renderer, cmark_node *node,
   case CMARK_NODE_FOOTNOTE_REFERENCE:
     if (entering) {
       LIT("[^");
-      OUT(cmark_chunk_to_cstr(renderer->mem, &node->as.literal), false, LITERAL);
+
+      char *footnote_label = renderer->mem->calloc(node->parent_footnote_def->as.literal.len + 1, sizeof(char));
+      memmove(footnote_label, node->parent_footnote_def->as.literal.data, node->parent_footnote_def->as.literal.len);
+
+      OUT(footnote_label, false, LITERAL);
+      renderer->mem->free(footnote_label);
+
       LIT("]");
     }
     break;
@@ -486,9 +492,13 @@ static int S_render_node(cmark_renderer *renderer, cmark_node *node,
     if (entering) {
       renderer->footnote_ix += 1;
       LIT("[^");
-      char n[32];
-      snprintf(n, sizeof(n), "%d", renderer->footnote_ix);
-      OUT(n, false, LITERAL);
+
+      char *footnote_label = renderer->mem->calloc(node->as.literal.len + 1, sizeof(char));
+      memmove(footnote_label, node->as.literal.data, node->as.literal.len);
+
+      OUT(footnote_label, false, LITERAL);
+      renderer->mem->free(footnote_label);
+
       LIT("]:\n");
 
       cmark_strbuf_puts(renderer->prefix, "    ");

data/ext/commonmarker/commonmarker.c

@@ -45,6 +45,13 @@ static VALUE encode_utf8_string(const char *c_string) {
   return string;
 }
 
+/* Encode a C string using the encoding from Ruby string +source+. */
+static VALUE encode_source_string(const char *c_string, VALUE source) {
+  VALUE string = rb_str_new2(c_string);
+  rb_enc_copy(string, source);
+  return string;
+}
+
 static void rb_mark_c_struct(void *data) {
   cmark_node *node = data;
   cmark_node *child;
@@ -108,25 +115,23 @@ static void rb_parent_removed(VALUE val) {
   RDATA(val)->dfree = rb_free_c_struct;
 }
 
-static cmark_parser *prepare_parser(VALUE rb_options, VALUE rb_extensions, cmark_mem *mem) {
+static cmark_parser *prepare_parser(VALUE rb_options, VALUE rb_extensions) {
   int options;
-  int extensions_len;
   VALUE rb_ext_name;
   int i;
 
-  Check_Type(rb_options, T_FIXNUM);
+  FIXNUM_P(rb_options);
+  options = FIX2INT(rb_options);
+
   Check_Type(rb_extensions, T_ARRAY);
 
-  options = FIX2INT(rb_options);
-  extensions_len = RARRAY_LEN(rb_extensions);
+  cmark_parser *parser = cmark_parser_new(options);
 
-  cmark_parser *parser = cmark_parser_new_with_mem(options, mem);
-  for (i = 0; i < extensions_len; ++i) {
-    rb_ext_name = RARRAY_PTR(rb_extensions)[i];
+  for (i = 0; i < RARRAY_LEN(rb_extensions); ++i) {
+    rb_ext_name = rb_ary_entry(rb_extensions, i);
 
     if (!SYMBOL_P(rb_ext_name)) {
       cmark_parser_free(parser);
-      cmark_arena_reset();
       rb_raise(rb_eTypeError, "extension names should be Symbols; got a %"PRIsVALUE"", rb_obj_class(rb_ext_name));
     }
 
@@ -135,7 +140,6 @@ static cmark_parser *prepare_parser(VALUE rb_options, VALUE rb_extensions, cmark
 
     if (!syntax_extension) {
       cmark_parser_free(parser);
-      cmark_arena_reset();
       rb_raise(rb_eArgError, "extension %s not found", rb_id2name(SYM2ID(rb_ext_name)));
     }
 
@@ -150,33 +154,57 @@ static cmark_parser *prepare_parser(VALUE rb_options, VALUE rb_extensions, cmark
  *
  */
 static VALUE rb_markdown_to_html(VALUE self, VALUE rb_text, VALUE rb_options, VALUE rb_extensions) {
-  char *str, *html;
-  int len;
+  char *html;
   cmark_parser *parser;
   cmark_node *doc;
+
   Check_Type(rb_text, T_STRING);
-  Check_Type(rb_options, T_FIXNUM);
 
-  parser = prepare_parser(rb_options, rb_extensions, cmark_get_arena_mem_allocator());
+  parser = prepare_parser(rb_options, rb_extensions);
 
-  str = (char *)RSTRING_PTR(rb_text);
-  len = RSTRING_LEN(rb_text);
+  cmark_parser_feed(parser, StringValuePtr(rb_text), RSTRING_LEN(rb_text));
+  doc = cmark_parser_finish(parser);
 
-  cmark_parser_feed(parser, str, len);
+  if (doc == NULL) {
+    cmark_parser_free(parser);
+    rb_raise(rb_eNodeError, "error parsing document");
+  }
+
+  html = cmark_render_html(doc, parser->options, parser->syntax_extensions);
+
+  cmark_parser_free(parser);
+  cmark_node_free(doc);
+
+  return rb_utf8_str_new_cstr(html);
+}
+
+/*
+ * Internal: Parses a Markdown string into an HTML string.
+ *
+ */
+static VALUE rb_markdown_to_xml(VALUE self, VALUE rb_text, VALUE rb_options, VALUE rb_extensions) {
+  char *xml;
+  cmark_parser *parser;
+  cmark_node *doc;
+
+  Check_Type(rb_text, T_STRING);
+
+  parser = prepare_parser(rb_options, rb_extensions);
+
+  cmark_parser_feed(parser, StringValuePtr(rb_text), RSTRING_LEN(rb_text));
   doc = cmark_parser_finish(parser);
+
   if (doc == NULL) {
-    cmark_arena_reset();
+    cmark_parser_free(parser);
     rb_raise(rb_eNodeError, "error parsing document");
   }
 
-  cmark_mem *default_mem = cmark_get_default_mem_allocator();
-  html = cmark_render_html_with_mem(doc, FIX2INT(rb_options), parser->syntax_extensions, default_mem);
-  cmark_arena_reset();
+  xml = cmark_render_xml(doc, parser->options);
 
-  VALUE ruby_html = rb_str_new2(html);
-  default_mem->free(html);
+  cmark_parser_free(parser);
+  cmark_node_free(doc);
 
-  return ruby_html;
+  return rb_utf8_str_new_cstr(xml);
 }
 
 /*
@@ -267,18 +295,17 @@ static VALUE rb_node_new(VALUE self, VALUE type) {
 static VALUE rb_parse_document(VALUE self, VALUE rb_text, VALUE rb_len,
                                VALUE rb_options, VALUE rb_extensions) {
   char *text;
-  int len, options;
+  int len;
   cmark_parser *parser;
   cmark_node *doc;
   Check_Type(rb_text, T_STRING);
   Check_Type(rb_len, T_FIXNUM);
   Check_Type(rb_options, T_FIXNUM);
 
-  parser = prepare_parser(rb_options, rb_extensions, cmark_get_default_mem_allocator());
+  parser = prepare_parser(rb_options, rb_extensions);
 
   text = (char *)RSTRING_PTR(rb_text);
   len = FIX2INT(rb_len);
-  options = FIX2INT(rb_options);
 
   cmark_parser_feed(parser, text, len);
   doc = cmark_parser_finish(parser);
@@ -567,6 +594,27 @@ static VALUE rb_render_html(VALUE self, VALUE rb_options, VALUE rb_extensions) {
   return ruby_html;
 }
 
+/* Internal: Convert the node to an XML string.
+ *
+ * Returns a {String}.
+ */
+static VALUE rb_render_xml(VALUE self, VALUE rb_options) {
+  int options;
+  cmark_node *node;
+  Check_Type(rb_options, T_FIXNUM);
+
+  options = FIX2INT(rb_options);
+
+  Data_Get_Struct(self, cmark_node, node);
+
+  char *xml = cmark_render_xml(node, options);
+  VALUE ruby_xml = rb_str_new2(xml);
+
+  free(xml);
+
+  return ruby_xml;
+}
+
 /* Internal: Convert the node to a CommonMark string.
  *
  * Returns a {String}.
@@ -1130,7 +1178,8 @@ static VALUE rb_html_escape_href(VALUE self, VALUE rb_text) {
   if (houdini_escape_href(&buf, (const uint8_t *)RSTRING_PTR(rb_text),
                           RSTRING_LEN(rb_text))) {
     result = (char *)cmark_strbuf_detach(&buf);
-    return rb_str_new2(result);
+    return encode_source_string(result, rb_text);
+
   }
 
   return rb_text;
@@ -1150,7 +1199,7 @@ static VALUE rb_html_escape_html(VALUE self, VALUE rb_text) {
   if (houdini_escape_html0(&buf, (const uint8_t *)RSTRING_PTR(rb_text),
                            RSTRING_LEN(rb_text), 0)) {
     result = (char *)cmark_strbuf_detach(&buf);
-    return rb_str_new2(result);
+    return encode_source_string(result, rb_text);
   }
 
   return rb_text;
@@ -1208,6 +1257,8 @@ __attribute__((visibility("default"))) void Init_commonmarker() {
   rb_cNode = rb_define_class_under(module, "Node", rb_cObject);
   rb_define_singleton_method(rb_cNode, "markdown_to_html", rb_markdown_to_html,
                              3);
+  rb_define_singleton_method(rb_cNode, "markdown_to_xml", rb_markdown_to_xml,
+                             3);
   rb_define_singleton_method(rb_cNode, "new", rb_node_new, 1);
   rb_define_singleton_method(rb_cNode, "parse_document", rb_parse_document, 4);
   rb_define_method(rb_cNode, "string_content", rb_node_get_string_content, 0);
@@ -1220,6 +1271,7 @@ __attribute__((visibility("default"))) void Init_commonmarker() {
   rb_define_method(rb_cNode, "next", rb_node_next, 0);
   rb_define_method(rb_cNode, "insert_before", rb_node_insert_before, 1);
   rb_define_method(rb_cNode, "_render_html", rb_render_html, 2);
+  rb_define_method(rb_cNode, "_render_xml", rb_render_xml, 1);
   rb_define_method(rb_cNode, "_render_commonmark", rb_render_commonmark, -1);
   rb_define_method(rb_cNode, "_render_plaintext", rb_render_plaintext, -1);
   rb_define_method(rb_cNode, "insert_after", rb_node_insert_after, 1);