commonmarker 0.10.0 → 0.11.0

data/ext/commonmarker/cmark/toolchain-mingw32.cmake

@@ -9,7 +9,7 @@ SET(CMAKE_RC_COMPILER i586-mingw32msvc-windres)
 # here is the target environment located
 SET(CMAKE_FIND_ROOT_PATH  /usr/i586-mingw32msvc "${CMAKE_SOURCE_DIR}/windows")
 
-# adjust the default behaviour of the FIND_XXX() commands:
+# adjust the default behaviour of the FIND_XYZ() commands:
 # search headers and libraries in the target environment, search 
 # programs in the host environment
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

data/ext/commonmarker/commonmarker.c

@@ -2,6 +2,10 @@
 #include "cmark.h"
 #include "houdini.h"
 #include "node.h"
+#include "registry.h"
+#include "parser.h"
+#include "syntax_extension.h"
+#include "core-extensions.h"
 
 static VALUE rb_mNodeError;
 static VALUE rb_mNode;
@@ -91,24 +95,69 @@ static void rb_parent_removed(VALUE val) {
   RDATA(val)->dfree = rb_free_c_struct;
 }
 
+static cmark_parser *prepare_parser(VALUE rb_options, VALUE rb_extensions) {
+  int options;
+  int extensions_len;
+  VALUE rb_ext_name;
+  int i;
+
+  Check_Type(rb_options, T_FIXNUM);
+  Check_Type(rb_extensions, T_ARRAY);
+
+  options = FIX2INT(rb_options);
+  extensions_len = RARRAY_LEN(rb_extensions);
+
+  cmark_parser *parser = cmark_parser_new(options);
+  for (i = 0; i < extensions_len; ++i) {
+    rb_ext_name = RARRAY_PTR(rb_extensions)[i];
+
+    if (!SYMBOL_P(rb_ext_name)) {
+      cmark_parser_free(parser);
+      rb_raise(rb_eTypeError, "extension names should be Symbols; got a %"PRIsVALUE"", rb_obj_class(rb_ext_name));
+    }
+
+    cmark_syntax_extension *syntax_extension =
+      cmark_find_syntax_extension(rb_id2name(SYM2ID(rb_ext_name)));
+
+    if (!syntax_extension) {
+      cmark_parser_free(parser);
+      rb_raise(rb_eArgError, "extension %s not found", rb_id2name(SYM2ID(rb_ext_name)));
+    }
+
+    cmark_parser_attach_syntax_extension(parser, syntax_extension);
+  }
+
+  return parser;
+}
+
 /*
  * Internal: Parses a Markdown string into an HTML string.
  *
  */
-static VALUE rb_markdown_to_html(VALUE self, VALUE rb_text, VALUE rb_options) {
-  char *str;
-  int len, options;
-
+static VALUE rb_markdown_to_html(VALUE self, VALUE rb_text, VALUE rb_options, VALUE rb_extensions) {
+  char *str, *html;
+  int len;
+  cmark_parser *parser;
+  cmark_node *doc;
   Check_Type(rb_text, T_STRING);
   Check_Type(rb_options, T_FIXNUM);
 
+  parser = prepare_parser(rb_options, rb_extensions);
+
   str = (char *)RSTRING_PTR(rb_text);
   len = RSTRING_LEN(rb_text);
-  options = FIX2INT(rb_options);
 
-  char *html = cmark_markdown_to_html(str, len, options);
-  VALUE ruby_html = rb_str_new2(html);
+  cmark_parser_feed(parser, str, len);
+  doc = cmark_parser_finish(parser);
+  if (doc == NULL) {
+    rb_raise(rb_mNodeError, "error parsing document");
+  }
 
+  html = cmark_render_html(doc, FIX2INT(rb_options), parser->syntax_extensions);
+  cmark_parser_free(parser);
+  cmark_node_free(doc);
+
+  VALUE ruby_html = rb_str_new2(html);
   free(html);
 
   return ruby_html;
@@ -196,22 +245,27 @@ static VALUE rb_node_new(VALUE self, VALUE type) {
  *
  */
 static VALUE rb_parse_document(VALUE self, VALUE rb_text, VALUE rb_len,
-                               VALUE rb_options) {
+                               VALUE rb_options, VALUE rb_extensions) {
   char *text;
   int len, options;
+  cmark_parser *parser;
   cmark_node *doc;
   Check_Type(rb_text, T_STRING);
   Check_Type(rb_len, T_FIXNUM);
   Check_Type(rb_options, T_FIXNUM);
 
+  parser = prepare_parser(rb_options, rb_extensions);
+
   text = (char *)RSTRING_PTR(rb_text);
   len = FIX2INT(rb_len);
   options = FIX2INT(rb_options);
 
-  doc = cmark_parse_document(text, len, options);
+  cmark_parser_feed(parser, text, len);
+  doc = cmark_parser_finish(parser);
   if (doc == NULL) {
     rb_raise(rb_mNodeError, "error parsing document");
   }
+  cmark_parser_free(parser);
 
   return rb_node_to_value(doc);
 }
@@ -439,23 +493,67 @@ static VALUE rb_node_insert_before(VALUE self, VALUE sibling) {
  *
  * Returns a {String}.
  */
-static VALUE rb_render_html(VALUE n, VALUE rb_options) {
-  int options;
+static VALUE rb_render_html(VALUE n, VALUE rb_options, VALUE rb_extensions) {
+  int options, extensions_len;
+  VALUE rb_ext_name;
+  int i;
   cmark_node *node;
+  cmark_llist *extensions = NULL;
   Check_Type(rb_options, T_FIXNUM);
+  Check_Type(rb_extensions, T_ARRAY);
 
   options = FIX2INT(rb_options);
+  extensions_len = RARRAY_LEN(rb_extensions);
 
   Data_Get_Struct(n, cmark_node, node);
 
-  char *html = cmark_render_html(node, options);
+  for (i = 0; i < extensions_len; ++i) {
+    rb_ext_name = RARRAY_PTR(rb_extensions)[i];
+
+    if (!SYMBOL_P(rb_ext_name)) {
+      cmark_llist_free(extensions);
+      rb_raise(rb_eTypeError, "extension names should be Symbols; got a %"PRIsVALUE"", rb_obj_class(rb_ext_name));
+    }
+
+    cmark_syntax_extension *syntax_extension =
+      cmark_find_syntax_extension(rb_id2name(SYM2ID(rb_ext_name)));
+
+    if (!syntax_extension) {
+      cmark_llist_free(extensions);
+      rb_raise(rb_eArgError, "extension %s not found\n", rb_id2name(SYM2ID(rb_ext_name)));
+    }
+
+    extensions = cmark_llist_append(extensions, syntax_extension);
+  }
+
+  char *html = cmark_render_html(node, options, extensions);
   VALUE ruby_html = rb_str_new2(html);
 
+  cmark_llist_free(extensions);
   free(html);
 
   return ruby_html;
 }
 
+/* Internal: Convert the node to a CommonMark string.
+ *
+ * Returns a {String}.
+ */
+static VALUE rb_render_commonmark(VALUE n, VALUE rb_options) {
+  int options;
+  cmark_node *node;
+  Check_Type(rb_options, T_FIXNUM);
+
+  options = FIX2INT(rb_options);
+  Data_Get_Struct(n, cmark_node, node);
+
+  char *cmark = cmark_render_commonmark(node, options, 120);
+  VALUE ruby_cmark = rb_str_new2(cmark);
+  free(cmark);
+
+  return ruby_cmark;
+}
+
 /*
  * Public: Inserts a node as a sibling after the current node.
  *
@@ -907,6 +1005,21 @@ static VALUE rb_html_escape_html(VALUE self, VALUE rb_text) {
   return rb_text;
 }
 
+VALUE rb_extensions(VALUE self) {
+  cmark_llist *exts, *it;
+  cmark_syntax_extension *ext;
+  VALUE ary = rb_ary_new();
+
+  exts = cmark_list_syntax_extensions();
+  for (it = exts; it; it = it->next) {
+    ext = it->data;
+    rb_ary_push(ary, rb_str_new2(ext->name));
+  }
+  cmark_llist_free(exts);
+
+  return ary;
+}
+
 __attribute__((visibility("default"))) void Init_commonmarker() {
   VALUE module;
   sym_document = ID2SYM(rb_intern("document"));
@@ -932,12 +1045,13 @@ __attribute__((visibility("default"))) void Init_commonmarker() {
   sym_ordered_list = ID2SYM(rb_intern("ordered_list"));
 
   module = rb_define_module("CommonMarker");
+  rb_define_singleton_method(module, "extensions", rb_extensions, 0);
   rb_mNodeError = rb_define_class_under(module, "NodeError", rb_eStandardError);
   rb_mNode = rb_define_class_under(module, "Node", rb_cObject);
   rb_define_singleton_method(rb_mNode, "markdown_to_html", rb_markdown_to_html,
-                             2);
+                             3);
   rb_define_singleton_method(rb_mNode, "new", rb_node_new, 1);
-  rb_define_singleton_method(rb_mNode, "parse_document", rb_parse_document, 3);
+  rb_define_singleton_method(rb_mNode, "parse_document", rb_parse_document, 4);
   rb_define_method(rb_mNode, "string_content", rb_node_get_string_content, 0);
   rb_define_method(rb_mNode, "string_content=", rb_node_set_string_content, 1);
   rb_define_method(rb_mNode, "type", rb_node_get_type, 0);
@@ -947,7 +1061,8 @@ __attribute__((visibility("default"))) void Init_commonmarker() {
   rb_define_method(rb_mNode, "first_child", rb_node_first_child, 0);
   rb_define_method(rb_mNode, "next", rb_node_next, 0);
   rb_define_method(rb_mNode, "insert_before", rb_node_insert_before, 1);
-  rb_define_method(rb_mNode, "_render_html", rb_render_html, 1);
+  rb_define_method(rb_mNode, "_render_html", rb_render_html, 2);
+  rb_define_method(rb_mNode, "_render_commonmark", rb_render_commonmark, 1);
   rb_define_method(rb_mNode, "insert_after", rb_node_insert_after, 1);
   rb_define_method(rb_mNode, "prepend_child", rb_node_prepend_child, 1);
   rb_define_method(rb_mNode, "append_child", rb_node_append_child, 1);
@@ -971,4 +1086,6 @@ __attribute__((visibility("default"))) void Init_commonmarker() {
 
   rb_define_method(rb_mNode, "html_escape_href", rb_html_escape_href, 1);
   rb_define_method(rb_mNode, "html_escape_html", rb_html_escape_html, 1);
+
+  cmark_register_plugin(core_extensions_registration);
 }

data/ext/commonmarker/commonmarker.h

@@ -1,6 +1,10 @@
 #ifndef COMMONMARKER_H
 #define COMMONMARKER_H
 
+#ifndef __MSXML_LIBRARY_DEFINED__
+#define __MSXML_LIBRARY_DEFINED__
+#endif
+
 #include "cmark.h"
 #include "ruby.h"
 

data/ext/commonmarker/extconf.rb

@@ -23,23 +23,30 @@ end
 FileUtils.mkdir_p(CMARK_BUILD_DIR)
 
 Dir.chdir(CMARK_BUILD_DIR) do
-  system 'cmake .. -DCMAKE_C_FLAGS=-fPIC'
-  system 'make libcmark_static'
+  if host_os == 'mingw32'
+    system 'cmake .. -G "MSYS Makefiles"'
+  else
+    system 'cmake .. -DCMAKE_C_FLAGS=-fPIC'
+  end
+  system "make libcmark_static" or abort "make libcmark_static failed"
+  system "make libcmarkextensions_static" or abort "make libcmarkextensions_static failed"
   # rake-compiler seems to complain about this line, not sure why it's messing with it
   FileUtils.rm_rf(File.join(CMARK_BUILD_DIR, 'Testing', 'Temporary'))
 end
 
 HEADER_DIRS = [INCLUDEDIR]
-LIB_DIRS = [LIBDIR, "#{CMARK_BUILD_DIR}/src"]
+LIB_DIRS = [LIBDIR, "#{CMARK_BUILD_DIR}/src", "#{CMARK_BUILD_DIR}/extensions"]
 
 dir_config('cmark', HEADER_DIRS, LIB_DIRS)
 
 # don't even bother to do this check if using OS X's messed up system Ruby: http://git.io/vsxkn
 unless sitearch =~ /^universal-darwin/
   abort 'libcmark is missing.' unless find_library('cmark', 'cmark_parse_document')
+  abort 'cmarkextensions is missing.' unless find_library('cmarkextensions', 'core_extensions_registration')
 end
 
-$LDFLAGS << " -L#{CMARK_BUILD_DIR}/src -lcmark"
-$CFLAGS << " -O2 -I#{CMARK_DIR}/src -I#{CMARK_BUILD_DIR}/src"
+$LDFLAGS << " -L#{CMARK_BUILD_DIR}/src -L#{CMARK_BUILD_DIR}/extensions -lcmark -lcmarkextensions"
+$CFLAGS << " -O2 -I#{CMARK_DIR}/src -I#{CMARK_DIR}/extensions -I#{CMARK_BUILD_DIR}/src"
+$CFLAGS << " -DCMARK_STATIC_DEFINE"
 
 create_makefile('commonmarker/commonmarker')

data/lib/commonmarker.rb

@@ -15,13 +15,14 @@ module CommonMarker
   #
   # text - A {String} of text
   # option - Either a {Symbol} or {Array of Symbol}s indicating the render options
+  # extensions - An {Array of Symbol}s indicating the extensions to use
   #
   # Returns a {String} of converted HTML.
-  def self.render_html(text, options = :default)
-    fail TypeError, 'text must be a string!' unless text.is_a?(String)
+  def self.render_html(text, options = :default, extensions = [])
+    fail TypeError, "text must be a String; got a #{text.class}!" unless text.is_a?(String)
     opts = Config.process_options(options, :render)
     text = text.encode('UTF-8')
-    html = Node.markdown_to_html(text, opts)
+    html = Node.markdown_to_html(text, opts, extensions)
     html.force_encoding('UTF-8')
   end
 
@@ -29,12 +30,13 @@ module CommonMarker
   #
   # string - {String} to be parsed
   # option - A {Symbol} or {Array of Symbol}s indicating the parse options
+  # extensions - An {Array of Symbol}s indicating the extensions to use
   #
   # Returns the `document` node.
-  def self.render_doc(text, options = :default)
-    fail TypeError, 'text must be a string!' unless text.is_a?(String)
+  def self.render_doc(text, options = :default, extensions = [])
+    fail TypeError, "text must be a String; got a #{text.class}!" unless text.is_a?(String)
     opts = Config.process_options(options, :parse)
     text = text.encode('UTF-8')
-    Node.parse_document(text, text.bytesize, opts)
+    Node.parse_document(text, text.bytesize, opts, extensions)
   end
 end

data/lib/commonmarker/node.rb

@@ -17,11 +17,22 @@ module CommonMarker
     # Public: Convert the node to an HTML string.
     #
     # options - A {Symbol} or {Array of Symbol}s indicating the render options
+    # extensions - An {Array of Symbol}s indicating the extensions to use
     #
     # Returns a {String}.
-    def to_html(options = :default)
+    def to_html(options = :default, extensions = [])
       opts = Config.process_options(options, :render)
-      _render_html(opts).force_encoding('utf-8')
+      _render_html(opts, extensions).force_encoding('utf-8')
+    end
+
+    # Public: Convert the node to a CommonMark string.
+    #
+    # options - A {Symbol} or {Array of Symbol}s indicating the render options
+    #
+    # Returns a {String}.
+    def to_commonmark(options = :default)
+      opts = Config.process_options(options, :render)
+      _render_commonmark(opts).force_encoding('utf-8')
     end
 
     # Public: Iterate over the children (if any) of the current pointer.

data/lib/commonmarker/renderer.rb

@@ -5,7 +5,7 @@ module CommonMarker
   class Renderer
     attr_accessor :in_tight, :warnings, :in_plain
     def initialize
-      @stream = StringIO.new
+      @stream = StringIO.new("".force_encoding("utf-8"))
       @need_blocksep = false
       @warnings = Set.new []
       @in_tight = false

data/lib/commonmarker/version.rb

@@ -1,3 +1,3 @@
 module CommonMarker
-  VERSION = '0.10.0'.freeze
+  VERSION = '0.11.0'.freeze
 end

data/test/test_commonmark.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class TestCommonmark < Minitest::Test
+  HTML_COMMENT = %r[<!--.*?-->\s?]
+
+  def setup
+    @markdown = <<-MD
+Hi *there*!
+
+1. I am a numeric list.
+2. I continue the list.
+* Suddenly, an unordered list!
+* What fun!
+
+Okay, _enough_.
+
+| a   | b   |
+| --- | --- |
+| c   | d   |
+    MD
+  end
+
+  def render_doc(doc)
+    CommonMarker.render_doc(doc, :default, %i[table])
+  end
+
+  def test_to_commonmark
+    compare = render_doc(@markdown).to_commonmark
+
+    assert_equal \
+      render_doc(@markdown).to_html.gsub(HTML_COMMENT, ''),
+      render_doc(compare).to_html.gsub(HTML_COMMENT, '')
+  end
+end

data/test/test_encoding.rb

@@ -1,9 +1,10 @@
+# coding: utf-8
 require 'test_helper'
 
 class TestEncoding < Minitest::Test
   # see http://git.io/vq4FR
   def test_encoding
-    contents = File.read(File.join(FIXTURES_DIR, 'curly.md'))
+    contents = File.read(File.join(FIXTURES_DIR, 'curly.md'), encoding: 'utf-8')
     doc = CommonMarker.render_doc(contents, :smart)
     render = doc.to_html
     assert_equal render.rstrip, '<p>This curly quote “makes commonmarker throw an exception”.</p>'

data/test/test_extensions.rb

@@ -0,0 +1,50 @@
+require 'test_helper'
+
+class TestExtensions < Minitest::Test
+  def setup
+    @markdown = <<-MD
+One extension:
+
+| a   | b   |
+| --- | --- |
+| c   | d   |
+| **x** | |
+
+Another extension:
+
+~~hi~~
+    MD
+  end
+
+  def test_uses_specified_extensions
+    CommonMarker.render_html(@markdown, :default, %i[]).tap do |out|
+      assert out.include?("| a")
+      assert out.include?("| <strong>x</strong>")
+      assert out.include?("~~hi~~")
+    end
+
+    CommonMarker.render_html(@markdown, :default, %i[table]).tap do |out|
+      refute out.include?("| a")
+      %w(<table> <tr> <th> a</th> <td> c</td> <strong>x</strong></td>).each {|html| assert out.include?(html) }
+      assert out.include?("~~hi~~")
+    end
+
+    CommonMarker.render_html(@markdown, :default, %i[strikethrough]).tap do |out|
+      assert out.include?("| a")
+      refute out.include?("~~hi~~")
+      assert out.include?("<del>hi</del>")
+    end
+
+    CommonMarker.render_html(@markdown, :default, %i[table strikethrough]).tap do |out|
+      refute out.include?("| a")
+      refute out.include?("| <strong>x</strong>")
+      refute out.include?("~~hi~~")
+    end
+  end
+
+  def test_bad_extension_specifications
+    assert_raises(TypeError) { CommonMarker.render_html(@markdown, :default, "nope") }
+    assert_raises(TypeError) { CommonMarker.render_html(@markdown, :default, ["table"]) }
+    assert_raises(ArgumentError) { CommonMarker.render_html(@markdown, :default, %i[table bad]) }
+  end
+end